From 4bd6ace2604281c1412545283b17fddc6b1fc684 Mon Sep 17 00:00:00 2001
From: ?? ? <ulysseskao@ximple.com.tw>
Date: Fri, 14 Mar 2008 14:22:41 +0800
Subject: [PATCH] update for EOFM-16
---
ximple-spatialjob/src/main/java/com/ximple/eofms/jobs/OracleConvertDgn2ShpJob.java | 251 +++++++++++++++++++++++++++++++++++++++++++------
1 files changed, 217 insertions(+), 34 deletions(-)
diff --git a/ximple-spatialjob/src/main/java/com/ximple/eofms/jobs/OracleConvertDgn2ShpJob.java b/ximple-spatialjob/src/main/java/com/ximple/eofms/jobs/OracleConvertDgn2ShpJob.java
index 3bed880..5dc43b3 100644
--- a/ximple-spatialjob/src/main/java/com/ximple/eofms/jobs/OracleConvertDgn2ShpJob.java
+++ b/ximple-spatialjob/src/main/java/com/ximple/eofms/jobs/OracleConvertDgn2ShpJob.java
@@ -1,19 +1,26 @@
package com.ximple.eofms.jobs;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Types;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
import java.util.Date;
-import java.util.TimeZone;
+import org.apache.commons.collections.OrderedMap;
+import org.apache.commons.collections.OrderedMapIterator;
+import org.apache.commons.collections.map.LinkedMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.geotools.feature.Feature;
+import org.geotools.feature.FeatureCollection;
import org.quartz.JobDetail;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
+
+import com.vividsolutions.jts.geom.GeometryFactory;
import oracle.jdbc.OracleConnection;
import oracle.jdbc.OracleResultSet;
@@ -22,6 +29,11 @@
import com.ximple.eofms.util.BinConverter;
import com.ximple.eofms.util.ByteArrayCompressor;
+import com.ximple.io.dgn7.ComplexElement;
+import com.ximple.io.dgn7.Dgn7fileException;
+import com.ximple.io.dgn7.Element;
+import com.ximple.io.dgn7.ElementType;
+import com.ximple.io.dgn7.IElementHandler;
import com.ximple.util.PrintfFormat;
/**
@@ -33,6 +45,19 @@
private static final int FETCHSIZE = 30;
private static final int BATCHSIZE = 25;
+ private static final int COMMITSIZE = 20;
+
+ class Pair
+ {
+ Object first;
+ Object second;
+
+ public Pair(Object first, Object second)
+ {
+ this.first = first;
+ this.second = second;
+ }
+ }
public void execute(JobExecutionContext context) throws JobExecutionException
{
@@ -63,37 +88,116 @@
private void exetcuteConvert(OracleConvertJobContext jobContext,
String querySchema, String dataPath) throws SQLException
{
- // verify igdsset_seed
- String srcTable = "IGSET_1";
- String destTable = OracleConvertJobContext.TABLE_PREFIX + "ELMSET_1";
- long startTime = System.currentTimeMillis();
- ArrayList srcElms = queryIgsetElement(jobContext, querySchema, srcTable);
- long currentTime = System.currentTimeMillis();
- SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss:SSS");
+ int order = 0;
+ OrderedMap map = getBlobStorageList(jobContext.getOracleConnection(), querySchema, "SD$SPACENODES"
+ , null);
- dateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
+ for (OrderedMapIterator it = map.orderedMapIterator(); it.hasNext();)
+ {
+ it.next();
- long elapsed = currentTime - startTime;
+ Pair pair = (Pair) it.getValue();
- System.out.println("Old Format:" + dateFormat.format(new Date(elapsed)));
- logger.info("query source element complete.");
- startTime = System.currentTimeMillis();
+ String tableSrc = (String) pair.first;
- ArrayList dstElms = queryRawElement(jobContext, querySchema, destTable);
+ queryIgsetElement(jobContext, querySchema, tableSrc);
- currentTime = System.currentTimeMillis();
- dateFormat = new SimpleDateFormat("HH:mm:ss:SSS");
- dateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
- elapsed = currentTime - startTime;
- System.out.println("New Format:" + dateFormat.format(new Date(elapsed)));
- logger.info("query dest element complete.");
- // Assert.isTrue(srcElms.size() == dstElms.size(), "src[" + srcElms.size() + "] != dest[" + dstElms.size() + "]");
+ order++;
+
+ if ((order % COMMITSIZE) == 0)
+ {
+ // OracleConnection connection = jobContext.getOracleConnection();
+ // connection.commit();
+ System.gc();
+ }
+ }
+
+ System.gc();
}
- private ArrayList queryIgsetElement(OracleConvertJobContext jobContext,
- String srcschema, String srctable) throws SQLException
+ private OrderedMap getBlobStorageList(OracleConnection connection, String schemaSrc, String tableSrc,
+ OrderedMap orderedMap) throws SQLException
{
- ArrayList result = new ArrayList();
+ if (orderedMap == null)
+ orderedMap = new LinkedMap(99);
+ String fetchStmtFmt = "SELECT SNID, SPACETABLE FROM \"%s\".\"%s\"";
+ PrintfFormat spf = new PrintfFormat(fetchStmtFmt);
+ String fetchStmt = spf.sprintf(new Object[]{schemaSrc, tableSrc});
+ Statement stmt = connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+
+ stmt.setFetchSize(FETCHSIZE);
+
+ ResultSet rs = stmt.executeQuery(fetchStmt);
+
+ while (rs.next())
+ {
+ int size = rs.getMetaData().getColumnCount();
+ Object[] values = new Object[size];
+
+ for (int i = 0; i < size; i++)
+ {
+ values[i] = rs.getObject(i + 1);
+ }
+
+ Integer key = new Integer(((BigDecimal) values[0]).intValue());
+ String name = (String) values[1];
+
+ Pair pair = (Pair) orderedMap.get(key);
+ if (pair == null)
+ orderedMap.put(key, new Pair(name, null));
+ else
+ pair.first = name;
+ }
+
+ rs.close();
+ stmt.close();
+
+ return orderedMap;
+ }
+
+ private OrderedMap getRawFormatStorageList(OracleConnection connection, String schemaSrc, String tableSrc,
+ OrderedMap orderedMap) throws SQLException
+ {
+ if (orderedMap == null)
+ orderedMap = new LinkedMap(99);
+ String fetchStmtFmt = "SELECT RNID, SPACETABLE FROM \"%s\".\"%s\"";
+ PrintfFormat spf = new PrintfFormat(fetchStmtFmt);
+ String fetchStmt = spf.sprintf(new Object[]{schemaSrc, tableSrc});
+ Statement stmt = connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+
+ stmt.setFetchSize(FETCHSIZE);
+
+ ResultSet rs = stmt.executeQuery(fetchStmt);
+
+ while (rs.next())
+ {
+ int size = rs.getMetaData().getColumnCount();
+ Object[] values = new Object[size];
+
+ for (int i = 0; i < size; i++)
+ {
+ values[i] = rs.getObject(i + 1);
+ }
+
+ Integer key = new Integer(((BigDecimal) values[0]).intValue());
+ String name = (String) values[1];
+
+ Pair pair = (Pair) orderedMap.get(key);
+ if (pair == null)
+ orderedMap.put(key, new Pair(null, name));
+ else
+ pair.second = name;
+ }
+
+ rs.close();
+ stmt.close();
+
+ return orderedMap;
+ }
+
+ private void queryIgsetElement(OracleConvertJobContext jobContext,
+ String srcschema, String srctable) throws SQLException
+ {
OracleConnection connection = jobContext.getOracleConnection();
String fetchSrcStmtFmt = "SELECT IGDSELM FROM \"%s\".\"%s\" ORDER BY ROWID";
PrintfFormat spf = new PrintfFormat(fetchSrcStmtFmt);
@@ -119,16 +223,19 @@
raw = rsSrc.getBytes(1);
}
- result.add(raw);
+ try
+ {
+ onBinaryElement(raw);
+ } catch (Dgn7fileException e)
+ {
+ logger.warn("Dgn7Exception", e);
+ }
}
-
- return result;
}
- private ArrayList queryRawElement(OracleConvertJobContext jobContext,
- String srcschema, String srctable) throws SQLException
+ private void queryRawElement(OracleConvertJobContext jobContext,
+ String srcschema, String srctable) throws SQLException
{
- ArrayList result = new ArrayList();
OracleConnection connection = jobContext.getOracleConnection();
String fetchDestStmtFmt = "SELECT ELEMENT FROM \"%s\".\"%s\" ORDER BY ROWID";
PrintfFormat spf = new PrintfFormat(fetchDestStmtFmt);
@@ -158,10 +265,81 @@
byte[] rawDest = ByteArrayCompressor.decompressByteArray(comparessedValue);
- result.add(rawDest);
+
+ try
+ {
+ onBinaryElement(rawDest);
+ } catch (Dgn7fileException e)
+ {
+ logger.warn("Dgn7Exception", e);
+ }
+ }
+ }
+
+ private void onBinaryElement(byte[] raws) throws Dgn7fileException
+ {
+ ByteBuffer buffer = ByteBuffer.wrap(raws);
+ buffer.order(ByteOrder.LITTLE_ENDIAN);
+ short signature = buffer.getShort();
+
+ // byte type = (byte) (buffer.get() & 0x7f);
+ byte type = (byte) ((signature >>> 8) & 0x007f);
+
+ // silly Bentley say contentLength is in 2-byte words
+ // and ByteByffer uses raws.
+ // track the record location
+ int elementLength = (buffer.getShort() * 2) + 4;
+ ElementType recordType = ElementType.forID(type);
+ IElementHandler handler = null;
+
+ handler = recordType.getElementHandler();
+
+ Element dgnElement = (Element) handler.read(buffer, signature, elementLength);
+ if (recordType.isComplexElement() && (elementLength < raws.length))
+ {
+ int offset = elementLength;
+ while (offset < (raws.length - 4))
+ {
+ buffer.position(offset);
+ signature = buffer.getShort();
+ type = (byte) ((signature >>> 8) & 0x007f);
+ elementLength = (buffer.getShort() * 2) + 4;
+ if (raws.length < (offset + elementLength))
+ {
+ System.out.println("Length not match:" + offset + ":" + buffer.position() + ":" + buffer.limit());
+ break;
+ }
+ recordType = ElementType.forID(type);
+ handler = recordType.getElementHandler();
+ if (handler != null)
+ {
+ Element subElement = (Element) handler.read(buffer, signature, elementLength);
+ ((ComplexElement) dgnElement).add(subElement);
+ offset += elementLength;
+ } else
+ {
+ byte[] remain = new byte[buffer.remaining()];
+ System.arraycopy(raws, offset, remain, 0, buffer.remaining());
+ for (int i = 0; i < remain.length; i++)
+ {
+ if (remain[i] != 0)
+ {
+ logger.info("fetch element has some error. index=" + (offset + i) + ":value=" + remain[i]);
+ System.out.println("fetch element has some error. index=" + (offset + i) + ":value=" + remain[i]);
+ }
+ }
+ break;
+ }
+ }
}
- return result;
+ Feature feature = convertElementToFeature(dgnElement);
+ }
+
+ private Feature convertElementToFeature(Element dgnElement)
+ {
+ GeometryFactory geomFactory = new GeometryFactory();
+ return null;
}
private boolean equalRawData(byte[] rawSrc, byte[] rawDest)
@@ -181,4 +359,9 @@
return true;
}
+
+ public FeatureCollection loopupFeatureCollection(String name)
+ {
+ return null;
+ }
}
--
Gitblit v0.0.0-SNAPSHOT