Merge branch 'origin/2.1.x'

Dennis Kao

2014-01-15 94ae08701bbd7585a0b7e5a92d1975965a503c03

 xdgnjobs/ximple-spatialjob/src/main/java/com/ximple/eofms/jobs/OracleConvertDgn2MySQLJob.java

@@ -6,14 +6,15 @@
import java.io.FilenameFilter;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.channels.FileChannel;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Types;
import java.sql.Connection;
import java.util.Date;
import java.util.Map;
import java.util.TreeMap;
@@ -24,16 +25,14 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.geotools.data.DataStore;
import org.geotools.data.mysql.MySQLDataStore;
import org.geotools.data.mysql.MySQLDataStoreFactory;
import org.geotools.feature.IllegalAttributeException;
import org.geotools.feature.SchemaException;
import org.geotools.jdbc.JDBCDataStore;
import org.opengis.feature.IllegalAttributeException;
import org.quartz.JobDataMap;
import org.quartz.JobDetail;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;

import com.vividsolutions.jts.geom.GeometryFactory;

import oracle.jdbc.OracleConnection;
import oracle.jdbc.OracleResultSet;
@@ -41,10 +40,10 @@
import oracle.sql.BLOB;

import com.ximple.eofms.jobs.context.AbstractOracleJobContext;
import com.ximple.eofms.jobs.context.mysql.OracleConvertMySQLJobContext;
import com.ximple.eofms.jobs.context.mysql.IndexDgnConvertMySQLJobContext;
import com.ximple.eofms.jobs.context.mysql.GeneralDgnConvertMySQLJobContext;
import com.ximple.eofms.jobs.context.mysql.FeatureDgnConvertMySQLJobContext;
import com.ximple.eofms.jobs.context.mysql.GeneralDgnConvertMySQLJobContext;
import com.ximple.eofms.jobs.context.mysql.IndexDgnConvertMySQLJobContext;
import com.ximple.eofms.jobs.context.mysql.OracleConvertMySQLJobContext;
import com.ximple.eofms.util.BinConverter;
import com.ximple.eofms.util.ByteArrayCompressor;
import com.ximple.eofms.util.StringUtils;
@@ -58,8 +57,7 @@
import com.ximple.io.dgn7.TextElement;
import com.ximple.util.PrintfFormat;

public class OracleConvertDgn2MySQLJob extends AbstractOracleDatabaseJob
{
public class OracleConvertDgn2MySQLJob extends AbstractOracleDatabaseJob {
    final static Log logger = LogFactory.getLog(OracleConvertDgn2PostGISJob.class);

    private static final String MYHOST = "MYHOST";
@@ -73,13 +71,11 @@
    private static final int FETCHSIZE = 30;
    private static final int COMMITSIZE = 20;

    class Pair
    {
    class Pair {
        Object first;
        Object second;

        public Pair(Object first, Object second)
        {
        public Pair(Object first, Object second) {
            this.first = first;
            this.second = second;
        }
@@ -87,7 +83,6 @@

    protected static MySQLDataStoreFactory dataStoreFactory = new MySQLDataStoreFactory();

    GeometryFactory _geomFactory = new GeometryFactory();
    protected String _myHost;
    protected String _myDatabase;
    protected String _myPort;
@@ -97,20 +92,19 @@
    protected String _myUseWKB;

    protected Map<String, String> myProperties;
    protected MySQLDataStore targetDataStore;
    protected JDBCDataStore targetDataStore;

    public Log getLogger()
    {
    public Log getLogger() {
        return logger;
    }

    protected AbstractOracleJobContext prepareJobContext(String targetSchemaName, String filterPath)
    {
        return new OracleConvertMySQLJobContext(getDataPath(), getTargetDataStore(), filterPath);
    protected AbstractOracleJobContext prepareJobContext(String targetSchemaName, String filterPath, boolean profileMode,
                                                         boolean useTransform) {
        return new OracleConvertMySQLJobContext(getDataPath(), getTargetDataStore(), filterPath, profileMode,
                                                useTransform);
    }

    protected void extractJobConfiguration(JobDetail jobDetail) throws JobExecutionException
    {
    protected void extractJobConfiguration(JobDetail jobDetail) throws JobExecutionException {
        super.extractJobConfiguration(jobDetail);
        JobDataMap dataMap = jobDetail.getJobDataMap();
        _myHost = dataMap.getString(MYHOST);
@@ -132,33 +126,27 @@
        logger.info("USEWKB=" + _myUseWKB);
        */

        if (_myHost == null)
        {
        if (_myHost == null) {
            logger.warn("MYHOST is null");
            throw new JobExecutionException("Unknown PostGIS host.");
        }
        if (_myDatabase == null)
        {
        if (_myDatabase == null) {
            logger.warn("PGDATABASE is null");
            throw new JobExecutionException("Unknown PostGIS database.");
        }
        if (_myPort == null)
        {
        if (_myPort == null) {
            logger.warn("MYPORT is null");
            throw new JobExecutionException("Unknown PostGIS port.");
        }
        if (_mySchema == null)
        {
        if (_mySchema == null) {
            logger.warn("MYSCHEMA is null");
            throw new JobExecutionException("Unknown PostGIS schema.");
        }
        if (_myUsername == null)
        {
        if (_myUsername == null) {
            logger.warn("PGUSERNAME is null");
            throw new JobExecutionException("Unknown PostGIS username.");
        }
        if (_myPassword == null)
        {
        if (_myPassword == null) {
            logger.warn("PGPASSWORD is null");
            throw new JobExecutionException("Unknown PostGIS password.");
        }
@@ -175,13 +163,12 @@
        myProperties = remote;
    }

    public void execute(JobExecutionContext context) throws JobExecutionException
    {
    public void execute(JobExecutionContext context) throws JobExecutionException {
        // Every job has its own job detail
        JobDetail jobDetail = context.getJobDetail();

        // The name is defined in the job definition
        String jobName = jobDetail.getName();
        String jobName = jobDetail.getKey().getName();
        String targetSchemaName = null;

        // Log the time the job started
@@ -189,36 +176,36 @@
        extractJobConfiguration(jobDetail);
        createSourceDataStore();
        createTargetDataStore();
        if (getSourceDataStore() == null)
        {
        if (getSourceDataStore() == null) {
            logger.warn("Cannot connect source oracle database.");
            throw new JobExecutionException("Cannot connect source oracle database.");
        }

        if (getTargetDataStore() == null)
        {
        if (getTargetDataStore() == null) {
            logger.warn("Cannot connect source postgreSQL database.");
            throw new JobExecutionException("Cannot connect source postgreSQL database.");
        }

        long t1 = System.currentTimeMillis();
        try
        {
        try {
            logger.info("-- step:clearOutputDatabase --");
            clearOutputDatabase();
            boolean bFirst = isCopyConnectivityMode();
            if (checkConvertDB())
            {
            if (checkConvertDB()) {
                logger.info("-- step:convertOracleDB --");

                for (String orgSchema : _orgSchema)
                {
                for (String orgSchema : _orgSchema) {
                    OracleConvertMySQLJobContext jobContext =
                            (OracleConvertMySQLJobContext) prepareJobContext(targetSchemaName, _filterPath);
                        (OracleConvertMySQLJobContext) prepareJobContext(targetSchemaName, _filterPath,
                                                                         isProfileMode(), isTransformed());
                    jobContext.setSourceDataStore(getSourceDataStore());
                    // jobContext.setConvertElementIn(_convertElementIn);
                    jobContext.setElementLogging(checkElementLogging());
                    jobContext.setExecutionContext(context);

                    fetchTPData(jobContext);
                    logger.info("TPC DIST:" + jobContext.getDistId() + ":" +
                        ((jobContext.getDistName() == null) ? "NULL" : jobContext.getDistName()));

                    if (bFirst)
                        copyConnectivity(jobContext);
@@ -233,22 +220,19 @@
                }
            }

            if (checkConvertFile())
            {
            if (checkConvertFile()) {
                logger.info("-- step:convertIndexDesignFile --");
                convertIndexDesignFile(context);
                logger.info("-- step:convertOtherDesignFile --");
                convertOtherDesignFile(context);
            }

            if (checkConvertElementIn())
            {
            if (checkConvertElementIn()) {
                logger.info("-- step:convertFeatureDesignFile --");
                convertFeatureDesignFile(context);
            }

            if (checkCreateDummy())
            {
            if (checkCreateDummy()) {
                logger.info("-- step:createDummyFeatureFile --");
                createDummyFeatureFile(context);
            }
@@ -258,38 +242,21 @@
            // public static final String DATE_FORMAT_NOW = "yyyy-MM-dd HH:mm:ss";
            // SimpleDateFormat sdf = new SimpleDateFormat(DATE_FORMAT_NOW);
            logger.warn("use time = " + ((t2 - t1) / 60000.0) + " min");
        } catch (SQLException e)
        {
        } catch (SQLException e) {
            logger.warn(e.getMessage(), e);
            throw new JobExecutionException("Database error. " + e.getMessage(), e);
        } catch (IOException ex)
        {
        } catch (IOException ex) {
            logger.warn(ex.getMessage(), ex);
            throw new JobExecutionException("IO error. " + ex.getMessage(), ex);
        }
        logger.warn(jobName + " end at " + new Date());
    }

    /**
     * Connectivity½Æ»s¤@­Óª©¥»¡A¦b¬d¸ß¹q¬y¤è¦V®É¥Î¨Ó¤ñ¹ïOMS¸ê®Æ®wªº¹q¾¹³s±µ©Ê(Connectivity)
     *
     * @param jobContext job context
     * @throws SQLException sql exception
     */
    private void copyConnectivity(OracleConvertMySQLJobContext jobContext) throws SQLException
    {
        Connection connection = jobContext.getOracleConnection();
        Statement stmt = connection.createStatement();
        stmt.execute(AbstractOracleJobContext.TRUNCATE_CONNECTIVITY_WEBCHECK);
        stmt.execute(AbstractOracleJobContext.COPY_CONNECTIVITY_TO_WEBCHECK);
    }

    private void exetcuteConvert(OracleConvertMySQLJobContext jobContext,
                                 String querySchema, String dataPath) throws SQLException
    {
                                 String querySchema, String dataPath) throws SQLException {
        int order = 0;
        OrderedMap map = getBlobStorageList(jobContext.getOracleConnection(), querySchema, "SD$SPACENODES"
                , null);
            , null);

        logger.info("begin convert job:[" + map.size() + "]:testmode=" + _testMode);

@@ -300,8 +267,7 @@
        //jobContext.startTransaction();
        jobContext.setCurrentSchema(querySchema);
        jobContext.getExecutionContext().put("ConvertDgn2PostGISJobProgress", 0);
        for (OrderedMapIterator it = map.orderedMapIterator(); it.hasNext();)
        {
        for (OrderedMapIterator it = map.orderedMapIterator(); it.hasNext();) {
            it.next();

            Pair pair = (Pair) it.getValue();
@@ -312,14 +278,12 @@

            order++;

            if (_testMode)
            {
            if (_testMode) {
                if ((_testCount < 0) || (order >= _testCount))
                    break;
            }

            if ((order % COMMITSIZE) == 0)
            {
            if ((order % COMMITSIZE) == 0) {
                // OracleConnection connection = jobContext.getOracleConnection();
                // connection.commitTransaction();
                jobContext.commitTransaction();
@@ -329,8 +293,7 @@
            }

            int now = order % step;
            if (now != current)
            {
            if (now != current) {
                current = now;
                jobContext.getExecutionContext().put("ConvertDgn2PostGISJobProgress", current);

@@ -346,8 +309,7 @@
    }

    protected OrderedMap getBlobStorageList(Connection connection, String schemaSrc, String tableSrc,
                                            OrderedMap orderedMap) throws SQLException
    {
                                            OrderedMap orderedMap) throws SQLException {
        if (orderedMap == null)
            orderedMap = new LinkedMap(99);
        String fetchStmtFmt = "SELECT SNID, SPACETABLE FROM \"%s\".\"%s\"";
@@ -358,16 +320,13 @@

        stmt.setFetchSize(FETCHSIZE);

        try
        {
        try {
            rs = stmt.executeQuery(fetchStmt);
            int size = rs.getMetaData().getColumnCount();
            while (rs.next())
            {
            while (rs.next()) {
                Object[] values = new Object[size];

                for (int i = 0; i < size; i++)
                {
                for (int i = 0; i < size; i++) {
                    values[i] = rs.getObject(i + 1);
                }

@@ -380,13 +339,11 @@
                else
                    pair.first = name;
            }
        } catch (SQLException e)
        {
        } catch (SQLException e) {
            logger.error(e.toString(), e);
            logger.error("stmt=" + fetchStmt);
            throw e;
        } finally
        {
        } finally {
            if (rs != null) rs.close();
            stmt.close();
        }
@@ -395,8 +352,7 @@
    }

    protected OrderedMap getRawFormatStorageList(OracleConnection connection, String schemaSrc, String tableSrc,
                                                 OrderedMap orderedMap) throws SQLException
    {
                                                 OrderedMap orderedMap) throws SQLException {
        if (orderedMap == null)
            orderedMap = new LinkedMap(99);
        String fetchStmtFmt = "SELECT RNID, SPACETABLE FROM \"%s\".\"%s\"";
@@ -408,12 +364,10 @@

        ResultSet rs = stmt.executeQuery(fetchStmt);
        int size = rs.getMetaData().getColumnCount();
        while (rs.next())
        {
        while (rs.next()) {
            Object[] values = new Object[size];

            for (int i = 0; i < size; i++)
            {
            for (int i = 0; i < size; i++) {
                values[i] = rs.getObject(i + 1);
            }

@@ -434,8 +388,7 @@
    }

    protected void queryIgsetElement(OracleConvertMySQLJobContext jobContext,
                                     String srcschema, String srctable) throws SQLException
    {
                                     String srcschema, String srctable) throws SQLException {
        Connection connection = jobContext.getOracleConnection();
        String fetchSrcStmtFmt = "SELECT IGDSELM FROM \"%s\".\"%s\" ORDER BY ROWID";
        PrintfFormat spf = new PrintfFormat(fetchSrcStmtFmt);
@@ -446,27 +399,28 @@

        ResultSet rsSrc = stmtSrc.executeQuery(fetchSrcStmt);
        int igdsMetaType = rsSrc.getMetaData().getColumnType(1);
        while (rsSrc.next())
        {
            byte[] raw;
        while (rsSrc.next()) {
            byte[] raw = null;

            if (igdsMetaType == Types.BLOB)
            {
            if (igdsMetaType == Types.BLOB) {
                BLOB blob = (BLOB) rsSrc.getBlob(1);

                raw = getBytesFromBLOB(blob);
                blob.close();
            } else
            {
                try {
                    raw = getBytesFromBLOB(blob);
                } catch (BufferOverflowException e) {
                    logger.warn("Wrong Element Structure-", e);
                } finally {
                    // blob.close();
                }
            } else {
                raw = rsSrc.getBytes(1);
            }

            try
            {
                Element element = fetchBinaryElement(raw);
                jobContext.putFeatureCollection(element);
            } catch (Dgn7fileException e)
            {
            try {
                if (raw != null) {
                    Element element = fetchBinaryElement(raw);
                    jobContext.putFeatureCollection(element);
                }
            } catch (Dgn7fileException e) {
                logger.warn("Dgn7Exception", e);
            }
        }
@@ -476,8 +430,7 @@
    }

    protected void queryRawElement(OracleConvertMySQLJobContext jobContext,
                                   String srcschema, String srctable) throws SQLException
    {
                                   String srcschema, String srctable) throws SQLException {
        Connection connection = jobContext.getOracleConnection();
        String fetchDestStmtFmt = "SELECT ELEMENT FROM \"%s\".\"%s\" ORDER BY ROWID";
        PrintfFormat spf = new PrintfFormat(fetchDestStmtFmt);
@@ -488,8 +441,7 @@

        ResultSet rsDest = stmtDest.executeQuery(fetchDestStmt);

        while (rsDest.next())
        {
        while (rsDest.next()) {
            ARRAY rawsValue = ((OracleResultSet) rsDest).getARRAY(1);
            long[] rawData = rawsValue.getLongArray();
            byte[] comparessedValue;
@@ -508,12 +460,10 @@
            byte[] rawDest = ByteArrayCompressor.decompressByteArray(comparessedValue);


            try
            {
            try {
                Element element = fetchBinaryElement(rawDest);
                jobContext.putFeatureCollection(element);
            } catch (Dgn7fileException e)
            {
            } catch (Dgn7fileException e) {
                logger.warn("Dgn7Exception:" + e.getMessage(), e);
            }
        }
@@ -523,8 +473,7 @@
    }

    // Binary to Element
    private Element fetchBinaryElement(byte[] raws) throws Dgn7fileException
    {
    private Element fetchBinaryElement(byte[] raws) throws Dgn7fileException {
        ByteBuffer buffer = ByteBuffer.wrap(raws);
        buffer.order(ByteOrder.LITTLE_ENDIAN);
        short signature = buffer.getShort();
@@ -542,35 +491,28 @@
        handler = recordType.getElementHandler();

        Element dgnElement = (Element) handler.read(buffer, signature, elementLength);
        if (recordType.isComplexElement() && (elementLength < raws.length))
        {
        if (recordType.isComplexElement() && (elementLength < raws.length)) {
            int offset = elementLength;
            while (offset < (raws.length - 4))
            {
            while (offset < (raws.length - 4)) {
                buffer.position(offset);
                signature = buffer.getShort();
                type = (byte) ((signature >>> 8) & 0x007f);
                elementLength = (buffer.getShort() * 2) + 4;
                if (raws.length < (offset + elementLength))
                {
                if (raws.length < (offset + elementLength)) {
                    System.out.println("Length not match:" + offset + ":" + buffer.position() + ":" + buffer.limit());
                    break;
                }
                recordType = ElementType.forID(type);
                handler = recordType.getElementHandler();
                if (handler != null)
                {
                if (handler != null) {
                    Element subElement = (Element) handler.read(buffer, signature, elementLength);
                    ((ComplexElement) dgnElement).add(subElement);
                    offset += elementLength;
                } else
                {
                } else {
                    byte[] remain = new byte[buffer.remaining()];
                    System.arraycopy(raws, offset, remain, 0, buffer.remaining());
                    for (int i = 0; i < remain.length; i++)
                    {
                        if (remain[i] != 0)
                        {
                    for (int i = 0; i < remain.length; i++) {
                        if (remain[i] != 0) {
                            logger.info("fetch element has some error. index=" + (offset + i) + ":value=" + remain[i]);
                            System.out.println("fetch element has some error. index=" + (offset + i) + ":value=" + remain[i]);
                        }
@@ -584,41 +526,35 @@
    }

    /**
     * °õ¦æÂà´«¯Á¤Þ¹ÏÀɪº¤u§@
     * �����ഫ���޹��ɪ��u�@
     *
     * @param context ¤u§@°õ¦æÀô¹Ò
     * @param context �u�@��������
     * @throws org.quartz.JobExecutionException
     *          exception
     */
    private void convertIndexDesignFile(JobExecutionContext context) throws JobExecutionException
    {
    private void convertIndexDesignFile(JobExecutionContext context) throws JobExecutionException {
        File indexDir = new File(getDataPath(), "index");
        if (!indexDir.exists())
        {
        if (!indexDir.exists()) {
            logger.info("index dir=" + indexDir + " not exist.");
            return;
        }

        if (!indexDir.isDirectory())
        {
        if (!indexDir.isDirectory()) {
            logger.info("index dir=" + indexDir + " is not a directory.");
        }

        File[] dgnFiles = indexDir.listFiles(new FilenameFilter()
        {
            public boolean accept(File dir, String name)
            {
        File[] dgnFiles = indexDir.listFiles(new FilenameFilter() {
            public boolean accept(File dir, String name) {
                return name.toLowerCase().endsWith(".dgn");
            }
        });

        for (File dgnFile : dgnFiles)
        {
        for (File dgnFile : dgnFiles) {
            IndexDgnConvertMySQLJobContext convertContext =
                    new IndexDgnConvertMySQLJobContext(getDataPath(), getTargetDataStore());
                new IndexDgnConvertMySQLJobContext(getDataPath(), getTargetDataStore(), isProfileMode(),
                                                   isTransformed());
            logger.debug("--- start dgnfile-" + dgnFile.toString() + " ---");
            try
            {
            try {
                convertContext.setExecutionContext(context);
                String dgnPaths[] = StringUtils.splitToArray(dgnFile.toString(), File.separator);
                convertContext.setFilename(dgnPaths[dgnPaths.length - 1]);
@@ -635,28 +571,23 @@
                convertContext.closeFeatureWriter();
                System.gc();
                System.runFinalization();
            } catch (FileNotFoundException e)
            {
            } catch (FileNotFoundException e) {
                convertContext.rollbackTransaction();
                logger.warn(e.getMessage(), e);
                throw new JobExecutionException(e.getMessage(), e);
            } catch (Dgn7fileException e)
            {
            } catch (Dgn7fileException e) {
                convertContext.rollbackTransaction();
                logger.warn(e.getMessage(), e);
                throw new JobExecutionException(e.getMessage(), e);
            } catch (IOException e)
            {
            } catch (IOException e) {
                convertContext.rollbackTransaction();
                logger.warn(e.getMessage(), e);
                throw new JobExecutionException(e.getMessage(), e);
            } catch (IllegalAttributeException e)
            {
            } catch (IllegalAttributeException e) {
                convertContext.rollbackTransaction();
                logger.warn(e.getMessage(), e);
                throw new JobExecutionException(e.getMessage(), e);
            } catch (SchemaException e)
            {
            } catch (SchemaException e) {
                convertContext.rollbackTransaction();
                logger.warn(e.getMessage(), e);
                throw new JobExecutionException(e.getMessage(), e);
@@ -665,99 +596,81 @@
    }

    protected void scanIndexDgnElement(IndexDgnConvertMySQLJobContext convertContext)
            throws Dgn7fileException, IOException, IllegalAttributeException, SchemaException
    {
        throws Dgn7fileException, IOException, IllegalAttributeException, SchemaException {
        Dgn7fileReader reader = convertContext.getReader();
        int count = 0;
        Element lastComplex = null;
        while (reader.hasNext())
        {
            Dgn7fileReader.Record record = reader.nextElement();
            if (record.element() != null)
            {
        while (reader.hasNext()) {
            Element.FileRecord record = reader.nextElement();
            if (record.element() != null) {
                Element element = (Element) record.element();
                ElementType type = element.getElementType();

                if ((!type.isComplexElement()) && (!element.isComponentElement()))
                {
               if (lastComplex != null)
               {
                       processIndexElement(lastComplex, convertContext);
                       lastComplex = null;
               }
                if ((!type.isComplexElement()) && (!element.isComponentElement())) {
                    if (lastComplex != null) {
                        processIndexElement(lastComplex, convertContext);
                        lastComplex = null;
                    }

                    processIndexElement(element, convertContext);
                } else if (element.isComponentElement())
                {
                    if (lastComplex != null)
                    {
                } else if (element.isComponentElement()) {
                    if (lastComplex != null) {
                        ((ComplexElement) lastComplex).add(element);
                    }
                } else if (type.isComplexElement())
                {
                    if (lastComplex != null)
                    {
                } else if (type.isComplexElement()) {
                    if (lastComplex != null) {
                        processIndexElement(lastComplex, convertContext);
                    }
               lastComplex = element;
                    lastComplex = element;
                }
            }
            count++;
        }

        if (lastComplex != null)
        {
        if (lastComplex != null) {
            processIndexElement(lastComplex, convertContext);
        }
        logger.debug("ElementRecord Count=" + count);
    }

    private void processIndexElement(Element element, IndexDgnConvertMySQLJobContext convertContext)
            throws IllegalAttributeException, SchemaException
    {
        if (element instanceof TextElement)
        {
        throws IllegalAttributeException, SchemaException {
        if (element instanceof TextElement) {
            convertContext.putFeatureCollection(element);
        }
    }


    /**
     * °õ¦æÂà´«¨ä¥L³]­p¹ÏÀɪº¤u§@
     * �����ഫ��L�]�p���ɪ��u�@
     *
     * @param context jobContext
     * @throws org.quartz.JobExecutionException
     *          exception
     */
    private void convertOtherDesignFile(JobExecutionContext context) throws JobExecutionException
    {
    private void convertOtherDesignFile(JobExecutionContext context) throws JobExecutionException {
        File otherDir = new File(getDataPath(), "other");
        if (!otherDir.exists())
        {
        if (!otherDir.exists()) {
            logger.info("other dir=" + otherDir + " not exist.");
            return;
        }

        if (!otherDir.isDirectory())
        {
        if (!otherDir.isDirectory()) {
            logger.info("other dir=" + otherDir + " is not a directory.");
        }

        File[] dgnFiles = otherDir.listFiles(new FilenameFilter()
        {
            public boolean accept(File dir, String name)
            {
        File[] dgnFiles = otherDir.listFiles(new FilenameFilter() {
            public boolean accept(File dir, String name) {
                return name.toLowerCase().endsWith(".dgn");
            }
        });

        for (File dgnFile : dgnFiles)
        {
        for (File dgnFile : dgnFiles) {
            GeneralDgnConvertMySQLJobContext convertContext =
                    new GeneralDgnConvertMySQLJobContext(getDataPath(), getTargetDataStore());
                new GeneralDgnConvertMySQLJobContext(getDataPath(), getTargetDataStore(), isProfileMode(),
                                                     isTransformed());
            logger.info("--- start dgnfile-" + dgnFile.toString() + " ---");
            try
            {
            try {
                convertContext.setExecutionContext(context);
                String dgnPaths[] = StringUtils.splitToArray(dgnFile.toString(), File.separator);
                convertContext.setFilename(dgnPaths[dgnPaths.length - 1]);
@@ -774,28 +687,23 @@
                convertContext.closeFeatureWriter();
                System.gc();
                System.runFinalization();
            } catch (FileNotFoundException e)
            {
            } catch (FileNotFoundException e) {
                convertContext.rollbackTransaction();
                logger.warn(e.getMessage(), e);
                throw new JobExecutionException(e.getMessage(), e);
            } catch (Dgn7fileException e)
            {
            } catch (Dgn7fileException e) {
                convertContext.rollbackTransaction();
                logger.warn(e.getMessage(), e);
                throw new JobExecutionException(e.getMessage(), e);
            } catch (IOException e)
            {
            } catch (IOException e) {
                convertContext.rollbackTransaction();
                logger.warn(e.getMessage(), e);
                throw new JobExecutionException(e.getMessage(), e);
            } catch (IllegalAttributeException e)
            {
            } catch (IllegalAttributeException e) {
                convertContext.rollbackTransaction();
                logger.warn(e.getMessage(), e);
                throw new JobExecutionException(e.getMessage(), e);
            } catch (SchemaException e)
            {
            } catch (SchemaException e) {
                convertContext.rollbackTransaction();
                logger.warn(e.getMessage(), e);
                throw new JobExecutionException(e.getMessage(), e);
@@ -804,63 +712,51 @@
    }

    public void scanOtherDgnElement(GeneralDgnConvertMySQLJobContext convertContext)
            throws Dgn7fileException, IOException, IllegalAttributeException, SchemaException
    {
        throws Dgn7fileException, IOException, IllegalAttributeException, SchemaException {
        Dgn7fileReader reader = convertContext.getReader();
        int count = 0;
        Element lastComplex = null;
        while (reader.hasNext())
        {
            Dgn7fileReader.Record record = reader.nextElement();
            if (record.element() != null)
            {
        while (reader.hasNext()) {
            Element.FileRecord record = reader.nextElement();
            if (record.element() != null) {
                Element element = (Element) record.element();
                ElementType type = element.getElementType();

                if ((!type.isComplexElement()) && (!element.isComponentElement()))
                {
               if (lastComplex != null)
               {
                       processOtherElement(lastComplex, convertContext);
                       lastComplex = null;
               }
                if ((!type.isComplexElement()) && (!element.isComponentElement())) {
                    if (lastComplex != null) {
                        processOtherElement(lastComplex, convertContext);
                        lastComplex = null;
                    }

                    processOtherElement(element, convertContext);
                } else if (element.isComponentElement())
                {
                    if (lastComplex != null)
                    {
                } else if (element.isComponentElement()) {
                    if (lastComplex != null) {
                        ((ComplexElement) lastComplex).add(element);
                    }
                } else if (type.isComplexElement())
                {
                    if (lastComplex != null)
                    {
                } else if (type.isComplexElement()) {
                    if (lastComplex != null) {
                        processOtherElement(lastComplex, convertContext);
                    }
               lastComplex = element;
                    lastComplex = element;
                }
            }
            count++;
        }

        if (lastComplex != null)
        {
        if (lastComplex != null) {
            processOtherElement(lastComplex, convertContext);
        }
        logger.debug("ElementRecord Count=" + count);
    }

    private void processOtherElement(Element element, GeneralDgnConvertMySQLJobContext convertContext)
            throws IllegalAttributeException, SchemaException
    {
        throws IllegalAttributeException, SchemaException {
        convertContext.putFeatureCollection(element);
    }

    private void clearOutputDatabase()
    {
    private void clearOutputDatabase() {
        /*
        File outDataPath = new File(getDataPath(), OracleConvertPostGISJobContext.SHPOUTPATH);
        File outDataPath = new File(getDataPath(), OracleConvertEdbGeoJobContext.SHPOUTPATH);
        if (outDataPath.exists() && outDataPath.isDirectory())
        {
            deleteFilesInPath(outDataPath);
@@ -878,33 +774,24 @@
        */
    }

    private void deleteFilesInPath(File outDataPath)
    {
    private void deleteFilesInPath(File outDataPath) {
        deleteFilesInPath(outDataPath, true);
    }

    private void deleteFilesInPath(File outDataPath, boolean removeSubDir)
    {
        if (!outDataPath.isDirectory())
        {
    private void deleteFilesInPath(File outDataPath, boolean removeSubDir) {
        if (!outDataPath.isDirectory()) {
            return;
        }
        File[] files = outDataPath.listFiles();
        for (File file : files)
        {
            if (file.isFile())
            {
                if (!file.delete())
                {
        for (File file : files) {
            if (file.isFile()) {
                if (!file.delete()) {
                    logger.info("Cannot delete file-" + file.toString());
                }
            } else if (file.isDirectory())
            {
            } else if (file.isDirectory()) {
                deleteFilesInPath(file, removeSubDir);
                if (removeSubDir)
                {
                    if (file.delete())
                    {
                if (removeSubDir) {
                    if (file.delete()) {
                        logger.info("Cannot delete dir-" + file.toString());
                    }
                }
@@ -912,35 +799,29 @@
        }
    }

    private void convertFeatureDesignFile(JobExecutionContext context) throws JobExecutionException
    {
    private void convertFeatureDesignFile(JobExecutionContext context) throws JobExecutionException {
        File elminDir = new File(getDataPath(), "elmin");
        if (!elminDir.exists())
        {
        if (!elminDir.exists()) {
            logger.info("elmin dir=" + elminDir + " not exist.");
            return;
        }

        if (!elminDir.isDirectory())
        {
        if (!elminDir.isDirectory()) {
            logger.info("elmin dir=" + elminDir + " is not a directory.");
        }

        File[] dgnFiles = elminDir.listFiles(new FilenameFilter()
        {
            public boolean accept(File dir, String name)
            {
        File[] dgnFiles = elminDir.listFiles(new FilenameFilter() {
            public boolean accept(File dir, String name) {
                return name.toLowerCase().endsWith(".dgn");
            }
        });

        for (File dgnFile : dgnFiles)
        {
        for (File dgnFile : dgnFiles) {
            FeatureDgnConvertMySQLJobContext convertContext =
                    new FeatureDgnConvertMySQLJobContext(getDataPath(), getTargetDataStore(), _filterPath);
                new FeatureDgnConvertMySQLJobContext(getDataPath(), getTargetDataStore(), _filterPath, isProfileMode(),
                                                     isTransformed());
            logger.info("--- start dgnfile-" + dgnFile.toString() + " ---");
            try
            {
            try {
                convertContext.setExecutionContext(context);
                String dgnPaths[] = StringUtils.splitToArray(dgnFile.toString(), File.separator);
                convertContext.setFilename(dgnPaths[dgnPaths.length - 1]);
@@ -957,28 +838,23 @@
                convertContext.closeFeatureWriter();
                System.gc();
                System.runFinalization();
            } catch (FileNotFoundException e)
            {
            } catch (FileNotFoundException e) {
                convertContext.rollbackTransaction();
                logger.warn(e.getMessage(), e);
                throw new JobExecutionException(e.getMessage(), e);
            } catch (Dgn7fileException e)
            {
            } catch (Dgn7fileException e) {
                convertContext.rollbackTransaction();
                logger.warn(e.getMessage(), e);
                throw new JobExecutionException(e.getMessage(), e);
            } catch (IOException e)
            {
            } catch (IOException e) {
                convertContext.rollbackTransaction();
                logger.warn(e.getMessage(), e);
                throw new JobExecutionException(e.getMessage(), e);
            } catch (IllegalAttributeException e)
            {
            } catch (IllegalAttributeException e) {
                convertContext.rollbackTransaction();
                logger.warn(e.getMessage(), e);
                throw new JobExecutionException(e.getMessage(), e);
            } catch (SchemaException e)
            {
            } catch (SchemaException e) {
                convertContext.rollbackTransaction();
                logger.warn(e.getMessage(), e);
                throw new JobExecutionException(e.getMessage(), e);
@@ -987,61 +863,49 @@
    }

    public void scanFeatureDgnElement(FeatureDgnConvertMySQLJobContext convertContext)
            throws Dgn7fileException, IOException, IllegalAttributeException, SchemaException
    {
        throws Dgn7fileException, IOException, IllegalAttributeException, SchemaException {
        Dgn7fileReader reader = convertContext.getReader();
        int count = 0;
        Element lastComplex = null;
        while (reader.hasNext())
        {
            Dgn7fileReader.Record record = reader.nextElement();
            if (record.element() != null)
            {
        while (reader.hasNext()) {
            Element.FileRecord record = reader.nextElement();
            if (record.element() != null) {
                Element element = (Element) record.element();
                ElementType type = element.getElementType();

                if ((!type.isComplexElement()) && (!element.isComponentElement()))
                {
               if (lastComplex != null)
               {
                       processFeatureElement(lastComplex, convertContext);
                       lastComplex = null;
               }
                if ((!type.isComplexElement()) && (!element.isComponentElement())) {
                    if (lastComplex != null) {
                        processFeatureElement(lastComplex, convertContext);
                        lastComplex = null;
                    }

                    processFeatureElement(element, convertContext);
                } else if (element.isComponentElement())
                {
                    if (lastComplex != null)
                    {
                } else if (element.isComponentElement()) {
                    if (lastComplex != null) {
                        ((ComplexElement) lastComplex).add(element);
                    }
                } else if (type.isComplexElement())
                {
                    if (lastComplex != null)
                    {
                } else if (type.isComplexElement()) {
                    if (lastComplex != null) {
                        processFeatureElement(lastComplex, convertContext);
                    }
               lastComplex = element;
                    lastComplex = element;
                }
            }
            count++;
        }

        if (lastComplex != null)
        {
        if (lastComplex != null) {
            processFeatureElement(lastComplex, convertContext);
        }
        logger.debug("ElementRecord Count=" + count);
    }

    private void processFeatureElement(Element element, FeatureDgnConvertMySQLJobContext convertContext)
            throws IllegalAttributeException, SchemaException
    {
        throws IllegalAttributeException, SchemaException {
        convertContext.putFeatureCollection(element);
    }

    private void createDummyFeatureFile(JobExecutionContext context) throws JobExecutionException
    {
    private void createDummyFeatureFile(JobExecutionContext context) throws JobExecutionException {
        /*
        DummyFeatureConvertShpJobContext convertContext = new DummyFeatureConvertShpJobContext(getDataPath(), _filterPath);
        try {
@@ -1056,15 +920,12 @@
        */
    }

    public DataStore getTargetDataStore()
    {
    public DataStore getTargetDataStore() {
        return targetDataStore;
    }

    protected void createTargetDataStore() throws JobExecutionException
    {
        if (targetDataStore != null)
        {
    protected void createTargetDataStore() throws JobExecutionException {
        if (targetDataStore != null) {
            targetDataStore.dispose();
            targetDataStore = null;
        }
@@ -1076,33 +937,37 @@
        }
        */

        if (!myProperties.containsKey("max connections" /*MySQLDataStoreFactory.MAXCONN.key */))
        {
            myProperties.put("max connections", "2");
        if (!myProperties.containsKey(MySQLDataStoreFactory.MAXCONN.key)) {
            myProperties.put(MySQLDataStoreFactory.MAXCONN.key, "2");
        }

        if (!myProperties.containsKey("min connections" /* MySQLDataStoreFactory.MINCONN.key */))
        {
            myProperties.put("min connections", "1");
        if (!myProperties.containsKey(MySQLDataStoreFactory.MINCONN.key)) {
            myProperties.put(MySQLDataStoreFactory.MINCONN.key, "1");
        }

        if (!myProperties.containsKey(MySQLDataStoreFactory.WKBENABLED.key))
        {
        /*
        if (!myProperties.containsKey(MySQLDataStoreFactory.WKBENABLED.key)) {
            myProperties.put(MySQLDataStoreFactory.WKBENABLED.key, "true");
        }
        */

        if (!dataStoreFactory.canProcess(myProperties))
        {
        if (!dataStoreFactory.canProcess(myProperties)) {
            getLogger().warn("cannot process properties-");
            throw new JobExecutionException("cannot process properties-");
        }
        try
        {
            targetDataStore = (MySQLDataStore) dataStoreFactory.createDataStore(myProperties);
        } catch (IOException e)
        {
        try {
            targetDataStore = dataStoreFactory.createDataStore(myProperties);
        } catch (IOException e) {
            getLogger().warn(e.getMessage(), e);
            throw new JobExecutionException(e.getMessage(), e);
        }
    }

    protected void disconnect() {
        super.disconnect();
        if (targetDataStore != null) {
            targetDataStore.dispose();
            targetDataStore = null;
        }
    }
}