xdgnjobs/ximple-spatialjob/src/main/java/com/ximple/eofms/jobs/OracleConvertDgn2MySQLJob.java
@@ -72,7 +72,6 @@ private static final String USEWKB = "USEWKB"; private static final int FETCHSIZE = 30; private static final int BATCHSIZE = 25; private static final int COMMITSIZE = 20; class Pair @@ -362,10 +361,9 @@ try { rs = stmt.executeQuery(fetchStmt); int size = rs.getMetaData().getColumnCount(); while (rs.next()) { int size = rs.getMetaData().getColumnCount(); Object[] values = new Object[size]; for (int i = 0; i < size; i++) @@ -409,10 +407,9 @@ stmt.setFetchSize(FETCHSIZE); ResultSet rs = stmt.executeQuery(fetchStmt); int size = rs.getMetaData().getColumnCount(); while (rs.next()) { int size = rs.getMetaData().getColumnCount(); Object[] values = new Object[size]; for (int i = 0; i < size; i++) @@ -448,12 +445,12 @@ stmtSrc.setFetchSize(FETCHSIZE); ResultSet rsSrc = stmtSrc.executeQuery(fetchSrcStmt); int igdsMetaType = rsSrc.getMetaData().getColumnType(1); while (rsSrc.next()) { byte[] raw = null; byte[] raw; if (rsSrc.getMetaData().getColumnType(1) == Types.BLOB) if (igdsMetaType == Types.BLOB) { BLOB blob = (BLOB) rsSrc.getBlob(1);
@@ -72,7 +72,6 @@ private static final String USEWKB = "USEWKB"; private static final int FETCHSIZE = 30; private static final int BATCHSIZE = 25; private static final int COMMITSIZE = 20; class Pair @@ -362,10 +361,9 @@ try { rs = stmt.executeQuery(fetchStmt); int size = rs.getMetaData().getColumnCount(); while (rs.next()) { int size = rs.getMetaData().getColumnCount(); Object[] values = new Object[size]; for (int i = 0; i < size; i++) @@ -409,10 +407,9 @@ stmt.setFetchSize(FETCHSIZE); ResultSet rs = stmt.executeQuery(fetchStmt); int size = rs.getMetaData().getColumnCount(); while (rs.next()) { int size = rs.getMetaData().getColumnCount(); Object[] values = new Object[size]; for (int i = 0; i < size; i++) @@ -448,12 +445,11 @@ stmtSrc.setFetchSize(FETCHSIZE); ResultSet rsSrc = stmtSrc.executeQuery(fetchSrcStmt); int igdsMetaType = rsSrc.getMetaData().getColumnType(1); while (rsSrc.next()) { byte[] raw = null; if (rsSrc.getMetaData().getColumnType(1) == Types.BLOB) byte[] raw; if (igdsMetaType == Types.BLOB) { BLOB blob = (BLOB) rsSrc.getBlob(1);
@@ -9,12 +9,11 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.FileChannel; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.sql.Types; import java.sql.Connection; import java.util.Calendar; import java.util.Date; import java.util.Map; import java.util.TreeMap; @@ -262,7 +261,7 @@ // public static final String DATE_FORMAT_NOW = "yyyy-MM-dd HH:mm:ss"; // SimpleDateFormat sdf = new SimpleDateFormat(DATE_FORMAT_NOW); logger.warn("use time = " + ((int) ((t2 - t1) / 60000.0)) + " min - " + (((int) ((t2 - t1) % 60000.0)) / 1000) + " sec"); (((int) ((t2 - t1) % 60000.0)) / 1000) + " sec"); } catch (SQLException e) { logger.warn(e.getMessage(), e); @@ -295,7 +294,7 @@ { int order = 0; OrderedMap map = getBlobStorageList(jobContext.getOracleConnection(), querySchema, "SD$SPACENODES" , null); querySchema, "SD$SPACENODES", null); logger.info("begin convert job:[" + map.size() + "]:testmode=" + _testMode); @@ -363,14 +362,13 @@ ResultSet rs = null; stmt.setFetchSize(FETCHSIZE); int size = rs.getMetaData().getColumnCount(); try { rs = stmt.executeQuery(fetchStmt); while (rs.next()) { int size = rs.getMetaData().getColumnCount(); Object[] values = new Object[size]; for (int i = 0; i < size; i++) @@ -415,29 +413,29 @@ ResultSet rs = stmt.executeQuery(fetchStmt); try { while (rs.next()) { int size = rs.getMetaData().getColumnCount(); Object[] values = new Object[size]; for (int i = 0; i < size; i++) while (rs.next()) { values[i] = rs.getObject(i + 1); Object[] values = new Object[size]; for (int i = 0; i < size; i++) { values[i] = rs.getObject(i + 1); } Integer key = ((BigDecimal) values[0]).intValue(); String name = (String) values[1]; Pair pair = (Pair) orderedMap.get(key); if (pair == null) orderedMap.put(key, new Pair(null, name)); else pair.second = name; } Integer key = ((BigDecimal) values[0]).intValue(); String name = (String) values[1]; Pair pair = (Pair) orderedMap.get(key); if (pair == null) orderedMap.put(key, new Pair(null, name)); else pair.second = name; } } finally { rs.close(); stmt.close(); rs.close(); stmt.close(); } return orderedMap; } @@ -453,12 +451,11 @@ stmtSrc.setFetchSize(FETCHSIZE); ResultSet rsSrc = stmtSrc.executeQuery(fetchSrcStmt); int igdsMetaType = rsSrc.getMetaData().getColumnType(1); while (rsSrc.next()) { byte[] raw = null; if (rsSrc.getMetaData().getColumnType(1) == Types.BLOB) byte[] raw; if (igdsMetaType == Types.BLOB) { BLOB blob = (BLOB) rsSrc.getBlob(1); @@ -497,39 +494,39 @@ try { while (rsDest.next()) { ARRAY rawsValue = ((OracleResultSet) rsDest).getARRAY(1); long[] rawData = rawsValue.getLongArray(); byte[] comparessedValue; /* if (dataMode == TransferTask.DataMode.Normal) while (rsDest.next()) { ARRAY rawsValue = ((OracleResultSet) rsDest).getARRAY(1); long[] rawData = rawsValue.getLongArray(); byte[] comparessedValue; /* if (dataMode == TransferTask.DataMode.Normal) { comparessedValue = BinConverter.unmarshalByteArray(rawData, true); } else { comparessedValue = BinConverter.unmarshalCompactByteArray(rawData); } */ comparessedValue = BinConverter.unmarshalByteArray(rawData, true); } else { comparessedValue = BinConverter.unmarshalCompactByteArray(rawData); byte[] rawDest = ByteArrayCompressor.decompressByteArray(comparessedValue); try { Element element = fetchBinaryElement(rawDest); jobContext.putFeatureCollection(element); } catch (Dgn7fileException e) { logger.warn("Dgn7Exception:" + e.getMessage(), e); } } */ comparessedValue = BinConverter.unmarshalByteArray(rawData, true); byte[] rawDest = ByteArrayCompressor.decompressByteArray(comparessedValue); try { Element element = fetchBinaryElement(rawDest); jobContext.putFeatureCollection(element); } catch (Dgn7fileException e) { logger.warn("Dgn7Exception:" + e.getMessage(), e); } } } finally { rsDest.close(); stmtDest.close(); rsDest.close(); stmtDest.close(); } }
@@ -264,10 +264,10 @@ try { rs = stmt.executeQuery(fetchStmt); int size = rs.getMetaData().getColumnCount(); while (rs.next()) { int size = rs.getMetaData().getColumnCount(); Object[] values = new Object[size]; for (int i = 0; i < size; i++) @@ -309,12 +309,11 @@ Statement stmt = connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); stmt.setFetchSize(FETCHSIZE); ResultSet rs = stmt.executeQuery(fetchStmt); int size = rs.getMetaData().getColumnCount(); while (rs.next()) { int size = rs.getMetaData().getColumnCount(); Object[] values = new Object[size]; for (int i = 0; i < size; i++) @@ -350,12 +349,12 @@ stmtSrc.setFetchSize(FETCHSIZE); ResultSet rsSrc = stmtSrc.executeQuery(fetchSrcStmt); int igdsMetaType = rsSrc.getMetaData().getColumnType(1); while (rsSrc.next()) { byte[] raw = null; byte[] raw; if (rsSrc.getMetaData().getColumnType(1) == Types.BLOB) if (igdsMetaType == Types.BLOB) { BLOB blob = (BLOB) rsSrc.getBlob(1);
@@ -11,6 +11,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.sql.Types; import java.util.ArrayList; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -45,6 +46,7 @@ private FileOutputStream fos = null; private FileChannel fch = null; private int logCount = 0; private ArrayList<byte[]> dgnFileHeader = null; public OracleElementLogger(Connection connection) { @@ -226,6 +228,17 @@ logger.warn("connection is null"); return; } if (dgnFileHeader != null) { for (byte[] raw : dgnFileHeader) { putElementIntoStream(raw); } return; } dgnFileHeader = new ArrayList<byte[]>(); String fetchSrcStmtFmt = "SELECT IGDSELM FROM \"%s\".\"%s\" ORDER BY ROWID"; PrintfFormat spf = new PrintfFormat(fetchSrcStmtFmt); String fetchSrcStmt = spf.sprintf(new Object[]{currentSchema, TAB_IGDSSEED}); @@ -249,9 +262,12 @@ if (raw != null) { dgnFileHeader.add(raw); putElementIntoStream(raw); } } rsSrc.close(); stmtSrc.close(); } private void putElementIntoStream(byte[] raw) throws IOException @@ -268,7 +284,7 @@ byte[] chunk = new byte[optimalSize]; InputStream is = blob.getBinaryStream(0); ByteBuffer buffer = null; // ByteBuffer.allocate(optimalSize); int len = 0; int len; try { @@ -312,4 +328,9 @@ fch = null; } } public boolean isSchemaChanged() { return schemaChanged; } } xdgnjobs/ximple-spatialjob/src/main/java/com/ximple/eofms/jobs/context/mysql/OracleConvertMySQLJobContext.java
@@ -1,43 +1,42 @@ package com.ximple.eofms.jobs.context.mysql; import java.io.File; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.sql.DriverManager; import java.sql.SQLException; import java.util.HashMap; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.net.URL; import java.net.MalformedURLException; import java.io.File; import java.io.UnsupportedEncodingException; import java.io.IOException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.transaction.util.LoggerFacade; import org.apache.commons.transaction.util.CommonsLoggingLogger; import org.apache.commons.transaction.memory.PessimisticMapWrapper; import org.apache.commons.digester.Digester; import org.apache.commons.digester.xmlrules.DigesterLoader; import org.geotools.data.FeatureWriter; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.transaction.memory.PessimisticMapWrapper; import org.apache.commons.transaction.util.CommonsLoggingLogger; import org.apache.commons.transaction.util.LoggerFacade; import org.geotools.data.DataStore; import org.geotools.data.FeatureWriter; import org.geotools.data.Transaction; import org.geotools.data.SchemaNotFoundException; import org.geotools.data.mysql.MySQLDataStoreFactory; import org.geotools.feature.Feature; import org.geotools.feature.FeatureType; import org.geotools.feature.SimpleFeature; import org.geotools.feature.IllegalAttributeException; import org.geotools.feature.SimpleFeature; import org.quartz.JobExecutionContext; import org.xml.sax.SAXException; import com.vividsolutions.jts.util.Assert; import com.ximple.eofms.jobs.OracleElementLogger; import com.ximple.eofms.filter.ElementDispatcher; import com.ximple.eofms.filter.AbstractFLinkageDispatchableFilter; import com.ximple.eofms.filter.ElementDispatcher; import com.ximple.eofms.jobs.OracleElementLogger; import com.ximple.io.dgn7.ComplexElement; import com.ximple.io.dgn7.Element; import com.ximple.io.dgn7.FrammeAttributeData; import com.ximple.io.dgn7.ComplexElement; public class OracleConvertMySQLJobContext extends AbstractOracleToMySQLJobContext { @@ -70,7 +69,7 @@ private JobExecutionContext executionContext; private String currentSchema = null; private String pgCurrentSchema = null; private String mysqlCurrentSchema = null; private boolean schemaChanged = false; // private String _convertElementIn = null; @@ -289,6 +288,11 @@ return elmLogger; } public boolean isSchemaChanged() { return schemaChanged; } public String getCurrentSchema() { return currentSchema; xdgnjobs/ximple-spatialjob/src/main/java/com/ximple/eofms/jobs/context/orasdo/OracleConvertOraSDOJobContext.java
@@ -69,7 +69,7 @@ private JobExecutionContext executionContext; private String currentSchema = null; private String pgCurrentSchema = null; private String oraCurrentSchema = null; private boolean schemaChanged = false; // private String _convertElementIn = null; @@ -229,10 +229,9 @@ } ArrayList features = (ArrayList) featuresContext.get(featureType); Iterator itFeature = features.iterator(); while (itFeature.hasNext()) for (Object feature1 : features) { Feature feature = (Feature) itFeature.next(); Feature feature = (Feature) feature1; ((SimpleFeature) writer.next()).setAttributes(feature.getAttributes(null)); } //writer.close(); @@ -288,6 +287,11 @@ return elmLogger; } public boolean isSchemaChanged() { return schemaChanged; } public String getCurrentSchema() { return currentSchema; xdgnjobs/ximple-spatialjob/src/main/java/com/ximple/eofms/jobs/context/postgis/AbstractOracleToPostGISJobContext.java
@@ -116,7 +116,7 @@ */ private static final int MAX_ALLOWED_VALUE = 10485760; protected static final int BATCHSIZE = 256; protected static final int BATCHSIZE = 512; /** * Well Known Text writer (from JTS). xdgnjobs/ximple-spatialjob/src/main/java/com/ximple/eofms/jobs/context/postgis/OracleConvertPostGISJobContext.java
@@ -5,11 +5,11 @@ import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Connection; import java.sql.Statement; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -18,28 +18,23 @@ import org.apache.commons.digester.xmlrules.DigesterLoader; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.transaction.memory.PessimisticMapWrapper; import org.apache.commons.transaction.util.CommonsLoggingLogger; import org.apache.commons.transaction.util.LoggerFacade; import org.geotools.data.DataStore; import org.geotools.data.FeatureWriter; import org.geotools.data.Transaction; import org.geotools.data.postgis.PostgisDataStoreFactory; import org.geotools.feature.Feature; import org.geotools.feature.FeatureType; import org.geotools.feature.IllegalAttributeException; import org.geotools.feature.SimpleFeature; import org.geotools.feature.FeatureTypeBuilder; import org.geotools.feature.SchemaException; import org.postgresql.util.PSQLException; import org.quartz.JobExecutionContext; import org.xml.sax.SAXException; import org.postgresql.util.PSQLException; import com.vividsolutions.jts.util.Assert; import com.ximple.eofms.filter.AbstractFLinkageDispatchableFilter; import com.ximple.eofms.filter.ElementDispatcher; import com.ximple.eofms.filter.CreateFeatureTypeEventListener; import com.ximple.eofms.filter.ElementDispatcher; import com.ximple.eofms.filter.FeatureTypeEvent; import com.ximple.eofms.jobs.OracleElementLogger; import com.ximple.io.dgn7.ComplexElement; @@ -47,7 +42,7 @@ import com.ximple.io.dgn7.FrammeAttributeData; public class OracleConvertPostGISJobContext extends AbstractOracleToPostGISJobContext implements CreateFeatureTypeEventListener implements CreateFeatureTypeEventListener { static Log logger = LogFactory.getLog(OracleConvertPostGISJobContext.class); static final LoggerFacade sLogger = new CommonsLoggingLogger(logger); @@ -210,9 +205,11 @@ private void updateDataStore() { Iterator<FeatureType> it = txFeaturesContext.keySet().iterator(); String currentStmt = null; try { Connection conn = getConnection(); boolean autoCommit = conn.getAutoCommit(); conn.setAutoCommit(true); while (it.hasNext()) { FeatureType featureType = it.next(); @@ -220,9 +217,6 @@ String bindingStmt = makePrepareInsertSql(featureType); ArrayList<Feature> features = txFeaturesContext.get(featureType); Connection conn = getConnection(); boolean autoCommit = conn.getAutoCommit(); conn.setAutoCommit(true); PreparedStatement pstmt = conn.prepareStatement(bindingStmt); for (Feature feature : features) @@ -249,7 +243,7 @@ for (int i = 0; i < feature.getNumberOfAttributes(); i++) { logger.info("attr[" + i + "]-" + ((feature.getAttribute(i) == null) ? " NULL" : feature.getAttribute(i).toString())); feature.getAttribute(i).toString())); } logger.error(e.getMessage(), e); } @@ -257,9 +251,9 @@ pstmt.close(); features.clear(); conn.setAutoCommit(autoCommit); logger.debug("End Save into PostGIS:" + featureType.getTypeName()); } conn.setAutoCommit(autoCommit); accumulate = 0; } catch (SQLException e) { @@ -354,7 +348,8 @@ stmt.execute(stmtText); stmt.close(); } } else { } else { deleteTable(conn, featureName); } conn.close(); @@ -387,4 +382,9 @@ } } } public boolean isSchemaChanged() { return schemaChanged; } }
