View Javadoc
1   /*
2   Copyright (c) 2005 Health Market Science, Inc.
3   
4   Licensed under the Apache License, Version 2.0 (the "License");
5   you may not use this file except in compliance with the License.
6   You may obtain a copy of the License at
7   
8       http://www.apache.org/licenses/LICENSE-2.0
9   
10  Unless required by applicable law or agreed to in writing, software
11  distributed under the License is distributed on an "AS IS" BASIS,
12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  See the License for the specific language governing permissions and
14  limitations under the License.
15  */
16  
17  package com.healthmarketscience.jackcess.impl;
18  
19  import java.io.File;
20  import java.io.FileNotFoundException;
21  import java.io.IOException;
22  import java.io.InputStream;
23  import java.io.RandomAccessFile;
24  import java.lang.ref.ReferenceQueue;
25  import java.lang.ref.WeakReference;
26  import java.nio.ByteBuffer;
27  import java.nio.channels.Channels;
28  import java.nio.channels.FileChannel;
29  import java.nio.channels.ReadableByteChannel;
30  import java.nio.charset.Charset;
31  import java.util.ArrayList;
32  import java.util.Arrays;
33  import java.util.Calendar;
34  import java.util.Collection;
35  import java.util.Collections;
36  import java.util.Date;
37  import java.util.EnumMap;
38  import java.util.HashMap;
39  import java.util.HashSet;
40  import java.util.Iterator;
41  import java.util.LinkedHashMap;
42  import java.util.List;
43  import java.util.Map;
44  import java.util.NoSuchElementException;
45  import java.util.Set;
46  import java.util.TimeZone;
47  import java.util.TreeSet;
48  import java.util.regex.Pattern;
49  
50  import com.healthmarketscience.jackcess.ColumnBuilder;
51  import com.healthmarketscience.jackcess.Cursor;
52  import com.healthmarketscience.jackcess.CursorBuilder;
53  import com.healthmarketscience.jackcess.DataType;
54  import com.healthmarketscience.jackcess.Database;
55  import com.healthmarketscience.jackcess.DatabaseBuilder;
56  import com.healthmarketscience.jackcess.Index;
57  import com.healthmarketscience.jackcess.IndexBuilder;
58  import com.healthmarketscience.jackcess.IndexCursor;
59  import com.healthmarketscience.jackcess.PropertyMap;
60  import com.healthmarketscience.jackcess.Relationship;
61  import com.healthmarketscience.jackcess.Row;
62  import com.healthmarketscience.jackcess.RuntimeIOException;
63  import com.healthmarketscience.jackcess.Table;
64  import com.healthmarketscience.jackcess.TableBuilder;
65  import com.healthmarketscience.jackcess.TableMetaData;
66  import com.healthmarketscience.jackcess.impl.query.QueryImpl;
67  import com.healthmarketscience.jackcess.query.Query;
68  import com.healthmarketscience.jackcess.util.CaseInsensitiveColumnMatcher;
69  import com.healthmarketscience.jackcess.util.ColumnValidatorFactory;
70  import com.healthmarketscience.jackcess.util.ErrorHandler;
71  import com.healthmarketscience.jackcess.util.LinkResolver;
72  import com.healthmarketscience.jackcess.util.ReadOnlyFileChannel;
73  import com.healthmarketscience.jackcess.util.SimpleColumnValidatorFactory;
74  import com.healthmarketscience.jackcess.util.TableIterableBuilder;
75  import org.apache.commons.lang.builder.ToStringBuilder;
76  import org.apache.commons.logging.Log;
77  import org.apache.commons.logging.LogFactory;
78  
79  
80  /**
81   *
82   * @author Tim McCune
83   * @usage _intermediate_class_
84   */
85  public class DatabaseImpl implements Database
86  {  
87    private static final Log LOG = LogFactory.getLog(DatabaseImpl.class);
88  
89    /** this is the default "userId" used if we cannot find existing info.  this
90        seems to be some standard "Admin" userId for access files */
91    private static final byte[] SYS_DEFAULT_SID = new byte[] {
92      (byte) 0xA6, (byte) 0x33};
93  
94    /** the default value for the resource path used to load classpath
95     *  resources.
96     */
97    public static final String DEFAULT_RESOURCE_PATH = 
98      "com/healthmarketscience/jackcess/";
99  
100   /** the resource path to be used when loading classpath resources */
101   static final String RESOURCE_PATH = 
102     System.getProperty(RESOURCE_PATH_PROPERTY, DEFAULT_RESOURCE_PATH);
103 
104   /** whether or not this jvm has "broken" nio support */
105   static final boolean BROKEN_NIO = Boolean.TRUE.toString().equalsIgnoreCase(
106       System.getProperty(BROKEN_NIO_PROPERTY));
107 
108   /** additional internal details about each FileFormat */
109   private static final Map<Database.FileFormat,FileFormatDetails> FILE_FORMAT_DETAILS =
110     new EnumMap<Database.FileFormat,FileFormatDetails>(Database.FileFormat.class);
111 
112   static {
113     addFileFormatDetails(FileFormat.V1997, null, JetFormat.VERSION_3);
114     addFileFormatDetails(FileFormat.GENERIC_JET4, null, JetFormat.VERSION_4);
115     addFileFormatDetails(FileFormat.V2000, "empty", JetFormat.VERSION_4);
116     addFileFormatDetails(FileFormat.V2003, "empty2003", JetFormat.VERSION_4);
117     addFileFormatDetails(FileFormat.V2007, "empty2007", JetFormat.VERSION_12);
118     addFileFormatDetails(FileFormat.V2010, "empty2010", JetFormat.VERSION_14);
119     addFileFormatDetails(FileFormat.MSISAM, null, JetFormat.VERSION_MSISAM);
120   }
121   
122   /** System catalog always lives on page 2 */
123   private static final int PAGE_SYSTEM_CATALOG = 2;
124   /** Name of the system catalog */
125   private static final String TABLE_SYSTEM_CATALOG = "MSysObjects";
126 
127   /** this is the access control bit field for created tables.  the value used
128       is equivalent to full access (Visual Basic DAO PermissionEnum constant:
129       dbSecFullAccess) */
130   private static final Integer SYS_FULL_ACCESS_ACM = 1048575;
131 
132   /** ACE table column name of the actual access control entry */
133   private static final String ACE_COL_ACM = "ACM";
134   /** ACE table column name of the inheritable attributes flag */
135   private static final String ACE_COL_F_INHERITABLE = "FInheritable";
136   /** ACE table column name of the relevant objectId */
137   private static final String ACE_COL_OBJECT_ID = "ObjectId";
138   /** ACE table column name of the relevant userId */
139   private static final String ACE_COL_SID = "SID";
140 
141   /** Relationship table column name of the column count */
142   private static final String REL_COL_COLUMN_COUNT = "ccolumn";
143   /** Relationship table column name of the flags */
144   private static final String REL_COL_FLAGS = "grbit";
145   /** Relationship table column name of the index of the columns */
146   private static final String REL_COL_COLUMN_INDEX = "icolumn";
147   /** Relationship table column name of the "to" column name */
148   private static final String REL_COL_TO_COLUMN = "szColumn";
149   /** Relationship table column name of the "to" table name */
150   private static final String REL_COL_TO_TABLE = "szObject";
151   /** Relationship table column name of the "from" column name */
152   private static final String REL_COL_FROM_COLUMN = "szReferencedColumn";
153   /** Relationship table column name of the "from" table name */
154   private static final String REL_COL_FROM_TABLE = "szReferencedObject";
155   /** Relationship table column name of the relationship */
156   private static final String REL_COL_NAME = "szRelationship";
157   
158   /** System catalog column name of the page on which system object definitions
159       are stored */
160   private static final String CAT_COL_ID = "Id";
161   /** System catalog column name of the name of a system object */
162   private static final String CAT_COL_NAME = "Name";
163   private static final String CAT_COL_OWNER = "Owner";
164   /** System catalog column name of a system object's parent's id */
165   private static final String CAT_COL_PARENT_ID = "ParentId";
166   /** System catalog column name of the type of a system object */
167   private static final String CAT_COL_TYPE = "Type";
168   /** System catalog column name of the date a system object was created */
169   private static final String CAT_COL_DATE_CREATE = "DateCreate";
170   /** System catalog column name of the date a system object was updated */
171   private static final String CAT_COL_DATE_UPDATE = "DateUpdate";
172   /** System catalog column name of the flags column */
173   private static final String CAT_COL_FLAGS = "Flags";
174   /** System catalog column name of the properties column */
175   static final String CAT_COL_PROPS = "LvProp";
176   /** System catalog column name of the remote database */
177   private static final String CAT_COL_DATABASE = "Database";
178   /** System catalog column name of the remote table name */
179   private static final String CAT_COL_FOREIGN_NAME = "ForeignName";
180 
181   /** top-level parentid for a database */
182   private static final int DB_PARENT_ID = 0xF000000;
183 
184   /** the maximum size of any of the included "empty db" resources */
185   private static final long MAX_EMPTYDB_SIZE = 350000L;
186 
187   /** this object is a "system" object */
188   static final int SYSTEM_OBJECT_FLAG = 0x80000000;
189   /** this object is another type of "system" object */
190   static final int ALT_SYSTEM_OBJECT_FLAG = 0x02;
191   /** this object is hidden */
192   public static final int HIDDEN_OBJECT_FLAG = 0x08;
193   /** all flags which seem to indicate some type of system object */
194   static final int SYSTEM_OBJECT_FLAGS = 
195     SYSTEM_OBJECT_FLAG | ALT_SYSTEM_OBJECT_FLAG;
196 
197   /** read-only channel access mode */
198   public static final String RO_CHANNEL_MODE = "r";
199   /** read/write channel access mode */
200   public static final String RW_CHANNEL_MODE = "rw";
201 
202   /** Name of the system object that is the parent of all tables */
203   private static final String SYSTEM_OBJECT_NAME_TABLES = "Tables";
204   /** Name of the system object that is the parent of all databases */
205   private static final String SYSTEM_OBJECT_NAME_DATABASES = "Databases";
206   /** Name of the system object that is the parent of all relationships */
207   private static final String SYSTEM_OBJECT_NAME_RELATIONSHIPS = "Relationships";
208   /** Name of the table that contains system access control entries */
209   private static final String TABLE_SYSTEM_ACES = "MSysACEs";
210   /** Name of the table that contains table relationships */
211   private static final String TABLE_SYSTEM_RELATIONSHIPS = "MSysRelationships";
212   /** Name of the table that contains queries */
213   private static final String TABLE_SYSTEM_QUERIES = "MSysQueries";
214   /** Name of the table that contains complex type information */
215   private static final String TABLE_SYSTEM_COMPLEX_COLS = "MSysComplexColumns";
216   /** Name of the main database properties object */
217   private static final String OBJECT_NAME_DB_PROPS = "MSysDb";
218   /** Name of the summary properties object */
219   private static final String OBJECT_NAME_SUMMARY_PROPS = "SummaryInfo";
220   /** Name of the user-defined properties object */
221   private static final String OBJECT_NAME_USERDEF_PROPS = "UserDefined";
222   /** System object type for table definitions */
223   static final Short TYPE_TABLE = 1;
224   /** System object type for query definitions */
225   private static final Short TYPE_QUERY = 5;
226   /** System object type for linked table definitions */
227   private static final Short TYPE_LINKED_TABLE = 6;
228   /** System object type for relationships */
229   private static final Short TYPE_RELATIONSHIP = 8;
230 
231   /** max number of table lookups to cache */
232   private static final int MAX_CACHED_LOOKUP_TABLES = 50;
233 
234   /** the columns to read when reading system catalog normally */
235   private static Collection<String> SYSTEM_CATALOG_COLUMNS =
236     new HashSet<String>(Arrays.asList(CAT_COL_NAME, CAT_COL_TYPE, CAT_COL_ID,
237                                       CAT_COL_FLAGS, CAT_COL_PARENT_ID));
238   /** the columns to read when finding table details */
239   private static Collection<String> SYSTEM_CATALOG_TABLE_DETAIL_COLUMNS =
240     new HashSet<String>(Arrays.asList(CAT_COL_NAME, CAT_COL_TYPE, CAT_COL_ID, 
241                                       CAT_COL_FLAGS, CAT_COL_PARENT_ID, 
242                                       CAT_COL_DATABASE, CAT_COL_FOREIGN_NAME));
243   /** the columns to read when getting object propertyes */
244   private static Collection<String> SYSTEM_CATALOG_PROPS_COLUMNS =
245     new HashSet<String>(Arrays.asList(CAT_COL_ID, CAT_COL_PROPS));
246 
247   /** regex matching characters which are invalid in identifier names */
248   private static final Pattern INVALID_IDENTIFIER_CHARS = 
249     Pattern.compile("[\\p{Cntrl}.!`\\]\\[]");
250   
251   /** the File of the database */
252   private final File _file;
253   /** the simple name of the database */
254   private final String _name;
255   /** Buffer to hold database pages */
256   private ByteBuffer _buffer;
257   /** ID of the Tables system object */
258   private Integer _tableParentId;
259   /** Format that the containing database is in */
260   private final JetFormat _format;
261   /**
262    * Cache map of UPPERCASE table names to page numbers containing their
263    * definition and their stored table name (max size
264    * MAX_CACHED_LOOKUP_TABLES).
265    */
266   private final Map<String, TableInfo> _tableLookup =
267     new LinkedHashMap<String, TableInfo>() {
268     private static final long serialVersionUID = 0L;
269     @Override
270     protected boolean removeEldestEntry(Map.Entry<String, TableInfo> e) {
271       return(size() > MAX_CACHED_LOOKUP_TABLES);
272     }
273   };
274   /** set of table names as stored in the mdb file, created on demand */
275   private Set<String> _tableNames;
276   /** Reads and writes database pages */
277   private final PageChannel _pageChannel;
278   /** System catalog table */
279   private TableImpl _systemCatalog;
280   /** utility table finder */
281   private TableFinder _tableFinder;
282   /** System access control entries table (initialized on first use) */
283   private TableImpl _accessControlEntries;
284   /** ID of the Relationships system object */
285   private Integer _relParentId;
286   /** SIDs to use for the ACEs added for new relationships */
287   private final List<byte[]> _newRelSIDs = new ArrayList<byte[]>();
288   /** System relationships table (initialized on first use) */
289   private TableImpl _relationships;
290   /** System queries table (initialized on first use) */
291   private TableImpl _queries;
292   /** System complex columns table (initialized on first use) */
293   private TableImpl _complexCols;
294   /** SIDs to use for the ACEs added for new tables */
295   private final List<byte[]> _newTableSIDs = new ArrayList<byte[]>();
296   /** optional error handler to use when row errors are encountered */
297   private ErrorHandler _dbErrorHandler;
298   /** the file format of the database */
299   private FileFormat _fileFormat;
300   /** charset to use when handling text */
301   private Charset _charset;
302   /** timezone to use when handling dates */
303   private TimeZone _timeZone;
304   /** language sort order to be used for textual columns */
305   private ColumnImpl.SortOrder _defaultSortOrder;
306   /** default code page to be used for textual columns (in some dbs) */
307   private Short _defaultCodePage;
308   /** the ordering used for table columns */
309   private Table.ColumnOrder _columnOrder;
310   /** whether or not enforcement of foreign-keys is enabled */
311   private boolean _enforceForeignKeys;
312   /** whether or not auto numbers can be directly inserted by the user */
313   private boolean _allowAutoNumInsert;
314   /** factory for ColumnValidators */
315   private ColumnValidatorFactory _validatorFactory = SimpleColumnValidatorFactory.INSTANCE;
316   /** cache of in-use tables */
317   private final TableCache _tableCache = new TableCache();
318   /** handler for reading/writing properteies */
319   private PropertyMaps.Handler _propsHandler;
320   /** ID of the Databases system object */
321   private Integer _dbParentId;
322   /** owner of objects we create */
323   private byte[] _newObjOwner;
324   /** core database properties */
325   private PropertyMaps _dbPropMaps;
326   /** summary properties */
327   private PropertyMaps _summaryPropMaps;
328   /** user-defined properties */
329   private PropertyMaps _userDefPropMaps;
330   /** linked table resolver */
331   private LinkResolver _linkResolver;
332   /** any linked databases which have been opened */
333   private Map<String,Database> _linkedDbs;
334   /** shared state used when enforcing foreign keys */
335   private final FKEnforcer.SharedState _fkEnforcerSharedState =
336     FKEnforcer.initSharedState();
337   /** Calendar for use interpreting dates/times in Columns */
338   private Calendar _calendar;
339 
340   /**
341    * Open an existing Database.  If the existing file is not writeable or the
342    * readOnly flag is {@code true}, the file will be opened read-only.
343    * @param mdbFile File containing the database
344    * @param readOnly iff {@code true}, force opening file in read-only
345    *                 mode
346    * @param channel  pre-opened FileChannel.  if provided explicitly, it will
347    *                 not be closed by this Database instance
348    * @param autoSync whether or not to enable auto-syncing on write.  if
349    *                 {@code true}, writes will be immediately flushed to disk.
350    *                 This leaves the database in a (fairly) consistent state
351    *                 on each write, but can be very inefficient for many
352    *                 updates.  if {@code false}, flushing to disk happens at
353    *                 the jvm's leisure, which can be much faster, but may
354    *                 leave the database in an inconsistent state if failures
355    *                 are encountered during writing.  Writes may be flushed at
356    *                 any time using {@link #flush}.
357    * @param charset  Charset to use, if {@code null}, uses default
358    * @param timeZone TimeZone to use, if {@code null}, uses default
359    * @param provider CodecProvider for handling page encoding/decoding, may be
360    *                 {@code null} if no special encoding is necessary
361    * @usage _advanced_method_
362    */
363   public static DatabaseImpl open(
364       File mdbFile, boolean readOnly, FileChannel channel,
365       boolean autoSync, Charset charset, TimeZone timeZone, 
366       CodecProvider provider)
367     throws IOException
368   {
369     boolean closeChannel = false;
370     if(channel == null) {
371       if(!mdbFile.exists() || !mdbFile.canRead()) {
372         throw new FileNotFoundException("given file does not exist: " + 
373                                         mdbFile);
374       }
375 
376       // force read-only for non-writable files
377       readOnly |= !mdbFile.canWrite();
378 
379       // open file channel
380       channel = openChannel(mdbFile, readOnly);
381       closeChannel = true;
382     }
383 
384     boolean success = false;
385     try {
386 
387       if(!readOnly) {
388 
389         // verify that format supports writing
390         JetFormat jetFormat = JetFormat.getFormat(channel);
391 
392         if(jetFormat.READ_ONLY) {
393           // wrap the channel with a read-only version to enforce
394           // non-writability
395           channel = new ReadOnlyFileChannel(channel);
396           readOnly = true;
397         }
398       }
399 
400       DatabaseImpl db = new DatabaseImpl(mdbFile, channel, closeChannel, autoSync,
401                                          null, charset, timeZone, provider);
402       success = true;
403       return db;
404 
405     } finally {
406       if(!success && closeChannel) {
407         // something blew up, shutdown the channel (quietly)
408         ByteUtil.closeQuietly(channel);
409       }
410     }
411   }
412   
413   /**
414    * Create a new Database for the given fileFormat
415    * @param fileFormat version of new database.
416    * @param mdbFile Location to write the new database to.  <b>If this file
417    *                already exists, it will be overwritten.</b>
418    * @param channel  pre-opened FileChannel.  if provided explicitly, it will
419    *                 not be closed by this Database instance
420    * @param autoSync whether or not to enable auto-syncing on write.  if
421    *                 {@code true}, writes will be immediately flushed to disk.
422    *                 This leaves the database in a (fairly) consistent state
423    *                 on each write, but can be very inefficient for many
424    *                 updates.  if {@code false}, flushing to disk happens at
425    *                 the jvm's leisure, which can be much faster, but may
426    *                 leave the database in an inconsistent state if failures
427    *                 are encountered during writing.  Writes may be flushed at
428    *                 any time using {@link #flush}.
429    * @param charset  Charset to use, if {@code null}, uses default
430    * @param timeZone TimeZone to use, if {@code null}, uses default
431    * @usage _advanced_method_
432    */
433   public static DatabaseImpl create(FileFormat fileFormat, File mdbFile, 
434                                     FileChannel channel, boolean autoSync,
435                                     Charset charset, TimeZone timeZone)
436     throws IOException
437   {
438     FileFormatDetails details = getFileFormatDetails(fileFormat);
439     if (details.getFormat().READ_ONLY) {
440       throw new IOException("File format " + fileFormat +       
441                             " does not support writing for " + mdbFile);
442     }
443     if(details.getEmptyFilePath() == null) {
444       throw new IOException("File format " + fileFormat +       
445                             " does not support file creation for " + mdbFile);
446     }
447 
448     boolean closeChannel = false;
449     if(channel == null) {
450       channel = openChannel(mdbFile, false);
451       closeChannel = true;
452     }
453 
454     boolean success = false;
455     try {
456       channel.truncate(0);
457       transferDbFrom(channel, getResourceAsStream(details.getEmptyFilePath()));
458       channel.force(true);
459       DatabaseImpl db = new DatabaseImpl(mdbFile, channel, closeChannel, autoSync, 
460                                          fileFormat, charset, timeZone, null);
461       success = true;
462       return db;
463     } finally {
464       if(!success && closeChannel) {
465         // something blew up, shutdown the channel (quietly)
466         ByteUtil.closeQuietly(channel);
467       }
468     }
469   }
470 
471   /**
472    * Package visible only to support unit tests via DatabaseTest.openChannel().
473    * @param mdbFile file to open
474    * @param readOnly true if read-only
475    * @return a FileChannel on the given file.
476    * @exception FileNotFoundException
477    *            if the mode is <tt>"r"</tt> but the given file object does
478    *            not denote an existing regular file, or if the mode begins
479    *            with <tt>"rw"</tt> but the given file object does not denote
480    *            an existing, writable regular file and a new regular file of
481    *            that name cannot be created, or if some other error occurs
482    *            while opening or creating the file
483    */
484   static FileChannel openChannel(final File mdbFile, final boolean readOnly)
485     throws FileNotFoundException
486   {
487     final String mode = (readOnly ? RO_CHANNEL_MODE : RW_CHANNEL_MODE);
488     return new RandomAccessFile(mdbFile, mode).getChannel();
489   }
490   
491   /**
492    * Create a new database by reading it in from a FileChannel.
493    * @param file the File to which the channel is connected 
494    * @param channel File channel of the database.  This needs to be a
495    *    FileChannel instead of a ReadableByteChannel because we need to
496    *    randomly jump around to various points in the file.
497    * @param autoSync whether or not to enable auto-syncing on write.  if
498    *                 {@code true}, writes will be immediately flushed to disk.
499    *                 This leaves the database in a (fairly) consistent state
500    *                 on each write, but can be very inefficient for many
501    *                 updates.  if {@code false}, flushing to disk happens at
502    *                 the jvm's leisure, which can be much faster, but may
503    *                 leave the database in an inconsistent state if failures
504    *                 are encountered during writing.  Writes may be flushed at
505    *                 any time using {@link #flush}.
506    * @param fileFormat version of new database (if known)
507    * @param charset Charset to use, if {@code null}, uses default
508    * @param timeZone TimeZone to use, if {@code null}, uses default
509    */
510   protected DatabaseImpl(File file, FileChannel channel, boolean closeChannel,
511                          boolean autoSync, FileFormat fileFormat, Charset charset,
512                          TimeZone timeZone, CodecProvider provider)
513     throws IOException
514   {
515     _file = file;
516     _name = getName(file);
517     _format = JetFormat.getFormat(channel);
518     _charset = ((charset == null) ? getDefaultCharset(_format) : charset);
519     _columnOrder = getDefaultColumnOrder();
520     _enforceForeignKeys = getDefaultEnforceForeignKeys();
521     _allowAutoNumInsert = getDefaultAllowAutoNumberInsert();
522     _fileFormat = fileFormat;
523     _pageChannel = new PageChannel(channel, closeChannel, _format, autoSync);
524     _timeZone = ((timeZone == null) ? getDefaultTimeZone() : timeZone);
525     if(provider == null) {
526       provider = DefaultCodecProvider.INSTANCE;
527     }
528     // note, it's slighly sketchy to pass ourselves along partially
529     // constructed, but only our _format and _pageChannel refs should be
530     // needed
531     _pageChannel.initialize(this, provider);
532     _buffer = _pageChannel.createPageBuffer();
533     readSystemCatalog();
534   }
535 
536   public File getFile() {
537     return _file;
538   }
539 
540   public String getName() {
541     return _name;
542   }
543 
544   /**
545    * @usage _advanced_method_
546    */
547   public PageChannel getPageChannel() {
548     return _pageChannel;
549   }
550 
551   /**
552    * @usage _advanced_method_
553    */
554   public JetFormat getFormat() {
555     return _format;
556   }
557   
558   /**
559    * @return The system catalog table
560    * @usage _advanced_method_
561    */
562   public TableImpl getSystemCatalog() {
563     return _systemCatalog;
564   }
565   
566   /**
567    * @return The system Access Control Entries table (loaded on demand)
568    * @usage _advanced_method_
569    */
570   public TableImpl getAccessControlEntries() throws IOException {
571     if(_accessControlEntries == null) {
572       _accessControlEntries = getRequiredSystemTable(TABLE_SYSTEM_ACES);
573     }
574     return _accessControlEntries;
575   }
576 
577   /**
578    * @return the complex column system table (loaded on demand)
579    * @usage _advanced_method_
580    */
581   public TableImpl getSystemComplexColumns() throws IOException {
582     if(_complexCols == null) {
583       _complexCols = getRequiredSystemTable(TABLE_SYSTEM_COMPLEX_COLS);
584     }
585     return _complexCols;
586   }
587 
588   public ErrorHandler getErrorHandler() {
589     return((_dbErrorHandler != null) ? _dbErrorHandler : ErrorHandler.DEFAULT);
590   }
591 
592   public void setErrorHandler(ErrorHandler newErrorHandler) {
593     _dbErrorHandler = newErrorHandler;
594   }    
595 
596   public LinkResolver getLinkResolver() {
597     return((_linkResolver != null) ? _linkResolver : LinkResolver.DEFAULT);
598   }
599 
600   public void setLinkResolver(LinkResolver newLinkResolver) {
601     _linkResolver = newLinkResolver;
602   }    
603 
604   public Map<String,Database> getLinkedDatabases() {
605     return ((_linkedDbs == null) ? Collections.<String,Database>emptyMap() : 
606             Collections.unmodifiableMap(_linkedDbs));
607   }
608 
609   public boolean isLinkedTable(Table table) throws IOException {
610 
611     if((table == null) || (this == table.getDatabase())) {
612       // if the table is null or this db owns the table, not linked
613       return false;
614     }
615 
616     // common case, local table name == remote table name
617     TableInfo tableInfo = lookupTable(table.getName());
618     if((tableInfo != null) && tableInfo.isLinked() &&
619        matchesLinkedTable(table, ((LinkedTableInfo)tableInfo).linkedTableName,
620                           ((LinkedTableInfo)tableInfo).linkedDbName)) {
621       return true;
622     }
623 
624     // but, the local table name may not match the remote table name, so we
625     // need to do a search if the common case fails
626     return _tableFinder.isLinkedTable(table);
627   }  
628 
629   private boolean matchesLinkedTable(Table table, String linkedTableName,
630                                      String linkedDbName) {
631     return (table.getName().equalsIgnoreCase(linkedTableName) &&
632             (_linkedDbs != null) &&
633             (_linkedDbs.get(linkedDbName) == table.getDatabase()));
634   }
635   
636   public TimeZone getTimeZone() {
637     return _timeZone;
638   }
639 
640   public void setTimeZone(TimeZone newTimeZone) {
641     if(newTimeZone == null) {
642       newTimeZone = getDefaultTimeZone();
643     }
644     _timeZone = newTimeZone;
645     // clear cached calendar when timezone is changed
646     _calendar = null;
647   }    
648 
649   public Charset getCharset()
650   {
651     return _charset;
652   }
653 
654   public void setCharset(Charset newCharset) {
655     if(newCharset == null) {
656       newCharset = getDefaultCharset(getFormat());
657     }
658     _charset = newCharset;
659   }
660 
661   public Table.ColumnOrder getColumnOrder() {
662     return _columnOrder;
663   }
664 
665   public void setColumnOrder(Table.ColumnOrder newColumnOrder) {
666     if(newColumnOrder == null) {
667       newColumnOrder = getDefaultColumnOrder();
668     }
669     _columnOrder = newColumnOrder;
670   }
671 
672   public boolean isEnforceForeignKeys() {
673     return _enforceForeignKeys;
674   }
675 
676   public void setEnforceForeignKeys(Boolean newEnforceForeignKeys) {
677     if(newEnforceForeignKeys == null) {
678       newEnforceForeignKeys = getDefaultEnforceForeignKeys();
679     }
680     _enforceForeignKeys = newEnforceForeignKeys;
681   }
682 
683   public boolean isAllowAutoNumberInsert() {
684     return _allowAutoNumInsert;
685   }
686 
687   public void setAllowAutoNumberInsert(Boolean allowAutoNumInsert) {
688     if(allowAutoNumInsert == null) {
689       allowAutoNumInsert = getDefaultAllowAutoNumberInsert();
690     }
691     _allowAutoNumInsert = allowAutoNumInsert;
692   }
693 
694 
695   public ColumnValidatorFactory getColumnValidatorFactory() {
696     return _validatorFactory;
697   }
698 
699   public void setColumnValidatorFactory(ColumnValidatorFactory newFactory) {
700     if(newFactory == null) {
701       newFactory = SimpleColumnValidatorFactory.INSTANCE;
702     }
703     _validatorFactory = newFactory;
704   }
705   
706   /**
707    * @usage _advanced_method_
708    */
709   FKEnforcer.SharedState getFKEnforcerSharedState() {
710     return _fkEnforcerSharedState;
711   }
712 
713   /**
714    * @usage _advanced_method_
715    */
716   Calendar getCalendar() {
717     if(_calendar == null) {
718       _calendar = DatabaseBuilder.toCompatibleCalendar(
719           Calendar.getInstance(_timeZone));
720     }
721     return _calendar;
722   }
723 
724   /**
725    * @returns the current handler for reading/writing properties, creating if
726    * necessary
727    */
728   private PropertyMaps.Handler getPropsHandler() {
729     if(_propsHandler == null) {
730       _propsHandler = new PropertyMaps.Handler(this);
731     }
732     return _propsHandler;
733   }
734 
735   public FileFormat getFileFormat() throws IOException {
736 
737     if(_fileFormat == null) {
738 
739       Map<String,FileFormat> possibleFileFormats =
740         getFormat().getPossibleFileFormats();
741 
742       if(possibleFileFormats.size() == 1) {
743 
744         // single possible format (null key), easy enough
745         _fileFormat = possibleFileFormats.get(null);
746 
747       } else {
748 
749         // need to check the "AccessVersion" property
750         String accessVersion = (String)getDatabaseProperties().getValue(
751             PropertyMap.ACCESS_VERSION_PROP);
752 
753         if(isBlank(accessVersion)) {
754           // no access version, fall back to "generic"
755           accessVersion = null;
756         }
757         
758         _fileFormat = possibleFileFormats.get(accessVersion);
759         
760         if(_fileFormat == null) {
761           throw new IllegalStateException(withErrorContext(
762                   "Could not determine FileFormat"));
763         }
764       }
765     }
766     return _fileFormat;
767   }
768 
769   /**
770    * @return a (possibly cached) page ByteBuffer for internal use.  the
771    *         returned buffer should be released using
772    *         {@link #releaseSharedBuffer} when no longer in use
773    */
774   private ByteBuffer takeSharedBuffer() {
775     // we try to re-use a single shared _buffer, but occassionally, it may be
776     // needed by multiple operations at the same time (e.g. loading a
777     // secondary table while loading a primary table).  this method ensures
778     // that we don't corrupt the _buffer, but instead force the second caller
779     // to use a new buffer.
780     if(_buffer != null) {
781       ByteBuffer curBuffer = _buffer;
782       _buffer = null;
783       return curBuffer;
784     }
785     return _pageChannel.createPageBuffer();
786   }
787 
788   /**
789    * Relinquishes use of a page ByteBuffer returned by
790    * {@link #takeSharedBuffer}.
791    */
792   private void releaseSharedBuffer(ByteBuffer buffer) {
793     // we always stuff the returned buffer back into _buffer.  it doesn't
794     // really matter if multiple values over-write, at the end of the day, we
795     // just need one shared buffer
796     _buffer = buffer;
797   }
798   
799   /**
800    * @return the currently configured database default language sort order for
801    *         textual columns
802    * @usage _intermediate_method_
803    */
804   public ColumnImpl.SortOrder getDefaultSortOrder() throws IOException {
805 
806     if(_defaultSortOrder == null) {
807       initRootPageInfo();
808     }
809     return _defaultSortOrder;
810   }
811 
812   /**
813    * @return the currently configured database default code page for textual
814    *         data (may not be relevant to all database versions)
815    * @usage _intermediate_method_
816    */
817   public short getDefaultCodePage() throws IOException {
818 
819     if(_defaultCodePage == null) {
820       initRootPageInfo();
821     }
822     return _defaultCodePage;
823   }
824 
825   /**
826    * Reads various config info from the db page 0.
827    */
828   private void initRootPageInfo() throws IOException {
829     ByteBuffer buffer = takeSharedBuffer();
830     try {
831       _pageChannel.readPage(buffer, 0);
832       _defaultSortOrder = ColumnImpl.readSortOrder(
833           buffer, _format.OFFSET_SORT_ORDER, _format);
834       _defaultCodePage = buffer.getShort(_format.OFFSET_CODE_PAGE);
835     } finally {
836       releaseSharedBuffer(buffer);
837     }
838   }
839   
840   /**
841    * @return a PropertyMaps instance decoded from the given bytes (always
842    *         returns non-{@code null} result).
843    * @usage _intermediate_method_
844    */
845   public PropertyMaps readProperties(byte[] propsBytes, int objectId,
846                                      RowIdImpl rowId)
847     throws IOException 
848   {
849     return getPropsHandler().read(propsBytes, objectId, rowId);
850   }
851   
852   /**
853    * Read the system catalog
854    */
855   private void readSystemCatalog() throws IOException {
856     _systemCatalog = readTable(TABLE_SYSTEM_CATALOG, PAGE_SYSTEM_CATALOG,
857                                SYSTEM_OBJECT_FLAGS);
858 
859     try {
860       _tableFinder = new DefaultTableFinder(
861           _systemCatalog.newCursor()
862             .setIndexByColumnNames(CAT_COL_PARENT_ID, CAT_COL_NAME)
863             .setColumnMatcher(CaseInsensitiveColumnMatcher.INSTANCE)
864             .toIndexCursor());
865     } catch(IllegalArgumentException e) {
866       if(LOG.isDebugEnabled()) {
867         LOG.debug(withErrorContext(
868                 "Could not find expected index on table " +
869                 _systemCatalog.getName()));
870       }
871       // use table scan instead
872       _tableFinder = new FallbackTableFinder(
873           _systemCatalog.newCursor()
874             .setColumnMatcher(CaseInsensitiveColumnMatcher.INSTANCE)
875             .toCursor());
876     }
877 
878     _tableParentId = _tableFinder.findObjectId(DB_PARENT_ID, 
879                                                SYSTEM_OBJECT_NAME_TABLES);
880 
881     if(_tableParentId == null) {  
882       throw new IOException(withErrorContext(
883               "Did not find required parent table id"));
884     }
885 
886     if (LOG.isDebugEnabled()) {
887       LOG.debug(withErrorContext(
888           "Finished reading system catalog.  Tables: " + getTableNames()));
889     }
890   }
891   
892   public Set<String> getTableNames() throws IOException {
893     if(_tableNames == null) {
894       _tableNames = getTableNames(true, false, true);
895     }
896     return _tableNames;
897   }
898 
899   public Set<String> getSystemTableNames() throws IOException {
900     return getTableNames(false, true, false);
901   }
902 
903   private Set<String> getTableNames(boolean normalTables, boolean systemTables,
904                                     boolean linkedTables)
905     throws IOException
906   {
907     Set<String> tableNames = new TreeSet<String>(String.CASE_INSENSITIVE_ORDER);
908     _tableFinder.getTableNames(tableNames, normalTables, systemTables,
909                                linkedTables);
910     return tableNames;
911   }
912 
913   public Iterator<Table> iterator() {
914     try {
915       return new TableIterator(getTableNames());
916     } catch(IOException e) {
917       throw new RuntimeIOException(e);
918     }
919   }
920 
921   public Iterator<Table> iterator(TableIterableBuilder builder) {
922     try {
923       return new TableIterator(getTableNames(builder.isIncludeNormalTables(),
924                                              builder.isIncludeSystemTables(),
925                                              builder.isIncludeLinkedTables()));
926     } catch(IOException e) {
927       throw new RuntimeIOException(e);
928     }
929   }
930 
931   public TableIterableBuilder newIterable() {
932     return new TableIterableBuilder(this);
933   }
934   
935   public TableImpl getTable(String name) throws IOException {
936     return getTable(name, false);
937   }
938 
939   public TableMetaData getTableMetaData(String name) throws IOException {
940     return getTableInfo(name, true);
941   }  
942 
943   /**
944    * @param tableDefPageNumber the page number of a table definition
945    * @return The table, or null if it doesn't exist
946    * @usage _advanced_method_
947    */
948   public TableImpl getTable(int tableDefPageNumber) throws IOException {
949 
950     // first, check for existing table
951     TableImpl table = _tableCache.get(tableDefPageNumber);
952     if(table != null) {
953       return table;
954     }
955     
956     // lookup table info from system catalog
957     Row objectRow = _tableFinder.getObjectRow(
958         tableDefPageNumber, SYSTEM_CATALOG_COLUMNS);
959     if(objectRow == null) {
960       return null;
961     }
962 
963     String name = objectRow.getString(CAT_COL_NAME);
964     int flags = objectRow.getInt(CAT_COL_FLAGS);
965 
966     return readTable(name, tableDefPageNumber, flags);
967   }
968 
969   /**
970    * @param name Table name
971    * @param includeSystemTables whether to consider returning a system table
972    * @return The table, or null if it doesn't exist
973    */
974   protected TableImpl getTable(String name, boolean includeSystemTables) 
975     throws IOException 
976   {
977     TableInfo tableInfo = getTableInfo(name, includeSystemTables);
978     return ((tableInfo != null) ? 
979             getTable(tableInfo, includeSystemTables) : null);
980   }
981 
982   private TableInfo getTableInfo(String name, boolean includeSystemTables) 
983     throws IOException 
984   {
985     TableInfo tableInfo = lookupTable(name);
986     
987     if ((tableInfo == null) || (tableInfo.pageNumber == null)) {
988       return null;
989     }
990     if(!includeSystemTables && tableInfo.isSystem()) {
991       return null;
992     }
993 
994     return tableInfo;
995   }
996 
997   private TableImpl getTable(TableInfo tableInfo, boolean includeSystemTables) 
998     throws IOException 
999   {
1000     if(tableInfo.isLinked()) {
1001 
1002       if(_linkedDbs == null) {
1003         _linkedDbs = new HashMap<String,Database>();
1004       }
1005 
1006       String linkedDbName = ((LinkedTableInfo)tableInfo).linkedDbName;
1007       String linkedTableName = ((LinkedTableInfo)tableInfo).linkedTableName;
1008       Database linkedDb = _linkedDbs.get(linkedDbName);
1009       if(linkedDb == null) {
1010         linkedDb = getLinkResolver().resolveLinkedDatabase(this, linkedDbName);
1011         _linkedDbs.put(linkedDbName, linkedDb);
1012       }
1013       
1014       return ((DatabaseImpl)linkedDb).getTable(linkedTableName, 
1015                                                includeSystemTables);
1016     }
1017 
1018     return readTable(tableInfo.tableName, tableInfo.pageNumber,
1019                      tableInfo.flags);
1020   }
1021   
1022   /**
1023    * Create a new table in this database
1024    * @param name Name of the table to create
1025    * @param columns List of Columns in the table
1026    * @deprecated use {@link TableBuilder} instead
1027    */
1028   @Deprecated
1029   public void createTable(String name, List<ColumnBuilder> columns)
1030     throws IOException
1031   {
1032     createTable(name, columns, null);
1033   }
1034 
1035   /**
1036    * Create a new table in this database
1037    * @param name Name of the table to create
1038    * @param columns List of Columns in the table
1039    * @param indexes List of IndexBuilders describing indexes for the table
1040    * @deprecated use {@link TableBuilder} instead
1041    */
1042   @Deprecated
1043   public void createTable(String name, List<ColumnBuilder> columns,
1044                           List<IndexBuilder> indexes)
1045     throws IOException
1046   {
1047     new TableBuilder(name)
1048       .addColumns(columns)
1049       .addIndexes(indexes)
1050       .toTable(this);
1051   }
1052 
1053   public void createLinkedTable(String name, String linkedDbName, 
1054                                 String linkedTableName)
1055     throws IOException
1056   {
1057     if(lookupTable(name) != null) {
1058       throw new IllegalArgumentException(withErrorContext(
1059           "Cannot create linked table with name of existing table '" + name +   
1060           "'"));
1061     }
1062 
1063     validateIdentifierName(name, getFormat().MAX_TABLE_NAME_LENGTH, "table");
1064     validateName(linkedDbName, DataType.MEMO.getMaxSize(), 
1065                  "linked database");
1066     validateIdentifierName(linkedTableName, getFormat().MAX_TABLE_NAME_LENGTH, 
1067                            "linked table");
1068 
1069     getPageChannel().startWrite();
1070     try {
1071       
1072       int linkedTableId = _tableFinder.getNextFreeSyntheticId();
1073 
1074       addNewTable(name, linkedTableId, TYPE_LINKED_TABLE, linkedDbName, 
1075                   linkedTableName);
1076 
1077     } finally {
1078       getPageChannel().finishWrite();
1079     }
1080   }
1081 
1082   /**
1083    * Adds a newly created table to the relevant internal database structures.
1084    */
1085   void addNewTable(String name, int tdefPageNumber, Short type, 
1086                    String linkedDbName, String linkedTableName) 
1087     throws IOException 
1088   {
1089     //Add this table to our internal list.
1090     addTable(name, Integer.valueOf(tdefPageNumber), type, linkedDbName,
1091              linkedTableName);
1092     
1093     //Add this table to system tables
1094     addToSystemCatalog(name, tdefPageNumber, type, linkedDbName, 
1095                        linkedTableName, _tableParentId);
1096     addToAccessControlEntries(tdefPageNumber, _tableParentId, _newTableSIDs);
1097   }
1098 
1099   public List<Relationship> getRelationships(Table table1, Table table2)
1100     throws IOException
1101   {
1102     return getRelationships((TableImpl)table1, (TableImpl)table2);
1103   }
1104 
1105   public List<Relationship> getRelationships(
1106       TableImpl table1, TableImpl table2)
1107     throws IOException
1108   {
1109     int nameCmp = table1.getName().compareTo(table2.getName());
1110     if(nameCmp == 0) {
1111       throw new IllegalArgumentException(withErrorContext(
1112               "Must provide two different tables"));
1113     }
1114     if(nameCmp > 0) {
1115       // we "order" the two tables given so that we will return a collection
1116       // of relationships in the same order regardless of whether we are given
1117       // (TableFoo, TableBar) or (TableBar, TableFoo).
1118       TableImpl tmp = table1;
1119       table1 = table2;
1120       table2 = tmp;
1121     }
1122       
1123     return getRelationshipsImpl(table1, table2, true);
1124   }
1125 
1126   public List<Relationship> getRelationships(Table table)
1127     throws IOException
1128   {
1129     if(table == null) {
1130       throw new IllegalArgumentException(withErrorContext("Must provide a table"));
1131     }
1132     // since we are getting relationships specific to certain table include
1133     // all tables
1134     return getRelationshipsImpl((TableImpl)table, null, true);
1135   }
1136       
1137   public List<Relationship> getRelationships()
1138     throws IOException
1139   {
1140     return getRelationshipsImpl(null, null, false);
1141   }
1142       
1143   public List<Relationship> getSystemRelationships()
1144     throws IOException
1145   {
1146     return getRelationshipsImpl(null, null, true);
1147   }
1148       
1149   private List<Relationship> getRelationshipsImpl(
1150       TableImpl table1, TableImpl table2, boolean includeSystemTables)
1151     throws IOException
1152   {
1153     initRelationships();
1154     
1155     List<Relationship> relationships = new ArrayList<Relationship>();
1156 
1157     if(table1 != null) {
1158       Cursor cursor = createCursorWithOptionalIndex(
1159           _relationships, REL_COL_FROM_TABLE, table1.getName());
1160       collectRelationships(cursor, table1, table2, relationships,
1161                            includeSystemTables);
1162       cursor = createCursorWithOptionalIndex(
1163           _relationships, REL_COL_TO_TABLE, table1.getName());
1164       collectRelationships(cursor, table2, table1, relationships,
1165                            includeSystemTables);
1166     } else {
1167       collectRelationships(new CursorBuilder(_relationships).toCursor(),
1168                            null, null, relationships, includeSystemTables);
1169     }
1170     
1171     return relationships;
1172   }
1173 
1174   RelationshipImpl writeRelationship(RelationshipCreator creator) 
1175     throws IOException
1176   {
1177     initRelationships();
1178     
1179     String name = createRelationshipName(creator);
1180     RelationshipImpl newRel = creator.createRelationshipImpl(name);
1181 
1182     ColumnImpl ccol = _relationships.getColumn(REL_COL_COLUMN_COUNT);
1183     ColumnImpl flagCol = _relationships.getColumn(REL_COL_FLAGS);
1184     ColumnImpl icol = _relationships.getColumn(REL_COL_COLUMN_INDEX);
1185     ColumnImpl nameCol = _relationships.getColumn(REL_COL_NAME);
1186     ColumnImpl fromTableCol = _relationships.getColumn(REL_COL_FROM_TABLE);
1187     ColumnImpl fromColCol = _relationships.getColumn(REL_COL_FROM_COLUMN);
1188     ColumnImpl toTableCol = _relationships.getColumn(REL_COL_TO_TABLE);
1189     ColumnImpl toColCol = _relationships.getColumn(REL_COL_TO_COLUMN);
1190 
1191     int numCols = newRel.getFromColumns().size();
1192     List<Object[]> rows = new ArrayList<Object[]>(numCols);
1193     for(int i = 0; i < numCols; ++i) {
1194       Object[] row = new Object[_relationships.getColumnCount()];
1195       ccol.setRowValue(row, numCols);
1196       flagCol.setRowValue(row, newRel.getFlags());
1197       icol.setRowValue(row, i);
1198       nameCol.setRowValue(row, name);
1199       fromTableCol.setRowValue(row, newRel.getFromTable().getName());
1200       fromColCol.setRowValue(row, newRel.getFromColumns().get(i).getName());
1201       toTableCol.setRowValue(row, newRel.getToTable().getName());
1202       toColCol.setRowValue(row, newRel.getToColumns().get(i).getName());
1203       rows.add(row);
1204     }
1205 
1206     getPageChannel().startWrite();
1207     try {
1208       
1209       int relObjId = _tableFinder.getNextFreeSyntheticId();
1210       _relationships.addRows(rows);
1211       addToSystemCatalog(name, relObjId, TYPE_RELATIONSHIP, null, null,
1212                          _relParentId);
1213       addToAccessControlEntries(relObjId, _relParentId, _newRelSIDs);
1214       
1215     } finally {
1216       getPageChannel().finishWrite();
1217     }
1218 
1219     return newRel;
1220   }
1221 
1222   private void initRelationships() throws IOException {
1223     // the relationships table does not get loaded until first accessed
1224     if(_relationships == null) {
1225       // need the parent id of the relationships objects
1226       _relParentId = _tableFinder.findObjectId(DB_PARENT_ID, 
1227                                                SYSTEM_OBJECT_NAME_RELATIONSHIPS);
1228       _relationships = getRequiredSystemTable(TABLE_SYSTEM_RELATIONSHIPS);
1229     }
1230   }
1231 
1232   private String createRelationshipName(RelationshipCreator creator)
1233     throws IOException 
1234   {
1235     // ensure that the final identifier name does not get too long
1236     // - the primary name is limited to ((max / 2) - 3)
1237     // - the total name is limited to (max - 3)
1238     int maxIdLen = getFormat().MAX_INDEX_NAME_LENGTH;
1239     int limit = (maxIdLen / 2) - 3;
1240     String origName = creator.getName();
1241     if (origName == null) {
1242       origName = creator.getPrimaryTable().getName();
1243       if(origName.length() > limit) {
1244         origName = origName.substring(0, limit);
1245       }
1246       origName += creator.getSecondaryTable().getName();
1247     }
1248     limit = maxIdLen - 3;
1249     if(origName.length() > limit) {
1250       origName = origName.substring(0, limit);
1251     }
1252 
1253     // now ensure name is unique
1254     Set<String> names = new HashSet<String>();
1255     
1256     // collect the names of all relationships for uniqueness check
1257     for(Row row :
1258           CursorImpl.createCursor(_systemCatalog).newIterable().setColumnNames(
1259               SYSTEM_CATALOG_COLUMNS))
1260     {
1261       String name = row.getString(CAT_COL_NAME);
1262       if (name != null && TYPE_RELATIONSHIP.equals(row.get(CAT_COL_TYPE))) {
1263         names.add(toLookupName(name));
1264       }
1265     }
1266 
1267     if(creator.hasReferentialIntegrity()) {
1268       // relationship name will also be index name in secondary table, so must
1269       // check those names as well
1270       for(Index idx : creator.getSecondaryTable().getIndexes()) {
1271         names.add(toLookupName(idx.getName()));
1272       } 
1273     }
1274 
1275     String baseName = toLookupName(origName);
1276     String name = baseName;
1277     int i = 0;
1278     while(names.contains(name)) {
1279       name = baseName + (++i);
1280     }
1281 
1282     return ((i == 0) ? origName : (origName + i));
1283   }
1284   
1285   public List<Query> getQueries() throws IOException
1286   {
1287     // the queries table does not get loaded until first accessed
1288     if(_queries == null) {
1289       _queries = getRequiredSystemTable(TABLE_SYSTEM_QUERIES);
1290     }
1291 
1292     // find all the queries from the system catalog
1293     List<Row> queryInfo = new ArrayList<Row>();
1294     Map<Integer,List<QueryImpl.Row>> queryRowMap = 
1295       new HashMap<Integer,List<QueryImpl.Row>>();
1296     for(Row row :
1297           CursorImpl.createCursor(_systemCatalog).newIterable().setColumnNames(
1298               SYSTEM_CATALOG_COLUMNS))
1299     {
1300       String name = row.getString(CAT_COL_NAME);
1301       if (name != null && TYPE_QUERY.equals(row.get(CAT_COL_TYPE))) {
1302         queryInfo.add(row);
1303         Integer id = row.getInt(CAT_COL_ID);
1304         queryRowMap.put(id, new ArrayList<QueryImpl.Row>());
1305       }
1306     }
1307 
1308     // find all the query rows
1309     for(Row row : CursorImpl.createCursor(_queries)) {
1310       QueryImpl.Row queryRow = new QueryImpl.Row(row);
1311       List<QueryImpl.Row> queryRows = queryRowMap.get(queryRow.objectId);
1312       if(queryRows == null) {
1313         LOG.warn(withErrorContext(
1314                      "Found rows for query with id " + queryRow.objectId +
1315                      " missing from system catalog"));
1316         continue;
1317       }
1318       queryRows.add(queryRow);
1319     }
1320 
1321     // lastly, generate all the queries
1322     List<Query> queries = new ArrayList<Query>();
1323     for(Row row : queryInfo) {
1324       String name = row.getString(CAT_COL_NAME);
1325       Integer id = row.getInt(CAT_COL_ID);
1326       int flags = row.getInt(CAT_COL_FLAGS);
1327       List<QueryImpl.Row> queryRows = queryRowMap.get(id);
1328       queries.add(QueryImpl.create(flags, name, queryRows, id));
1329     }
1330 
1331     return queries;
1332   }
1333 
1334   public TableImpl getSystemTable(String tableName) throws IOException
1335   {
1336     return getTable(tableName, true);
1337   }
1338 
1339   private TableImpl getRequiredSystemTable(String tableName) throws IOException
1340   {
1341     TableImpl table = getSystemTable(tableName);
1342     if(table == null) { 
1343       throw new IOException(withErrorContext(
1344               "Could not find system table " + tableName));
1345     } 
1346     return table;
1347   }
1348 
1349   public PropertyMap getDatabaseProperties() throws IOException {
1350     if(_dbPropMaps == null) {
1351       _dbPropMaps = getPropertiesForDbObject(OBJECT_NAME_DB_PROPS);
1352     }
1353     return _dbPropMaps.getDefault();
1354   }
1355 
1356   public PropertyMap getSummaryProperties() throws IOException {
1357     if(_summaryPropMaps == null) {
1358       _summaryPropMaps = getPropertiesForDbObject(OBJECT_NAME_SUMMARY_PROPS);
1359     }
1360     return _summaryPropMaps.getDefault();
1361   }
1362 
1363   public PropertyMap getUserDefinedProperties() throws IOException {
1364     if(_userDefPropMaps == null) {
1365       _userDefPropMaps = getPropertiesForDbObject(OBJECT_NAME_USERDEF_PROPS);
1366     }
1367     return _userDefPropMaps.getDefault();
1368   }
1369 
1370   /**
1371    * @return the PropertyMaps for the object with the given id
1372    * @usage _advanced_method_
1373    */
1374   public PropertyMaps getPropertiesForObject(int objectId)
1375     throws IOException
1376   {
1377     Row objectRow = _tableFinder.getObjectRow(
1378         objectId, SYSTEM_CATALOG_PROPS_COLUMNS);
1379     byte[] propsBytes = null;
1380     RowIdImpl rowId = null;
1381     if(objectRow != null) {
1382       propsBytes = objectRow.getBytes(CAT_COL_PROPS);
1383       rowId = (RowIdImpl)objectRow.getId();
1384     }
1385     return readProperties(propsBytes, objectId, rowId);
1386   }
1387 
1388   private Integer getDbParentId() throws IOException {
1389     if(_dbParentId == null) {
1390       // need the parent id of the databases objects
1391       _dbParentId = _tableFinder.findObjectId(DB_PARENT_ID, 
1392                                               SYSTEM_OBJECT_NAME_DATABASES);
1393       if(_dbParentId == null) {  
1394         throw new IOException(withErrorContext(
1395                 "Did not find required parent db id"));
1396       }
1397     }
1398     return _dbParentId;
1399   }
1400 
1401   private byte[] getNewObjectOwner() throws IOException {
1402     if(_newObjOwner == null) {
1403       // there doesn't seem to be any obvious way to find the main "owner" of
1404       // an access db, but certain db objects seem to have the common db
1405       // owner.  we attempt to grab the db properties object and use its
1406       // owner.
1407       Row msysDbRow = _tableFinder.getObjectRow(
1408           getDbParentId(), OBJECT_NAME_DB_PROPS,
1409           Collections.singleton(CAT_COL_OWNER));
1410       byte[] owner = null;
1411       if(msysDbRow != null) {
1412         owner = msysDbRow.getBytes(CAT_COL_OWNER);
1413       }
1414       _newObjOwner = (((owner != null) && (owner.length > 0)) ? 
1415                       owner : SYS_DEFAULT_SID);
1416     }
1417     return _newObjOwner;
1418   }
1419 
1420   /**
1421    * @return property group for the given "database" object
1422    */
1423   private PropertyMaps getPropertiesForDbObject(String dbName)
1424     throws IOException
1425   {
1426     Row objectRow = _tableFinder.getObjectRow(
1427         getDbParentId(), dbName, SYSTEM_CATALOG_PROPS_COLUMNS);
1428     byte[] propsBytes = null;
1429     int objectId = -1;
1430     RowIdImpl rowId = null;
1431     if(objectRow != null) {
1432       propsBytes = objectRow.getBytes(CAT_COL_PROPS);
1433       objectId = objectRow.getInt(CAT_COL_ID);
1434       rowId = (RowIdImpl)objectRow.getId();
1435     }
1436     return readProperties(propsBytes, objectId, rowId);
1437   }
1438 
1439   public String getDatabasePassword() throws IOException
1440   {
1441     ByteBuffer buffer = takeSharedBuffer();
1442     try {
1443       _pageChannel.readPage(buffer, 0);
1444 
1445       byte[] pwdBytes = new byte[_format.SIZE_PASSWORD];
1446       buffer.position(_format.OFFSET_PASSWORD);
1447       buffer.get(pwdBytes);
1448 
1449       // de-mask password using extra password mask if necessary (the extra
1450       // password mask is generated from the database creation date stored in
1451       // the header)
1452       byte[] pwdMask = getPasswordMask(buffer, _format);
1453       if(pwdMask != null) {
1454         for(int i = 0; i < pwdBytes.length; ++i) {
1455           pwdBytes[i] ^= pwdMask[i % pwdMask.length];
1456         }
1457       }
1458     
1459       boolean hasPassword = false;
1460       for(int i = 0; i < pwdBytes.length; ++i) {
1461         if(pwdBytes[i] != 0) {
1462           hasPassword = true;
1463           break;
1464         }
1465       }
1466 
1467       if(!hasPassword) {
1468         return null;
1469       }
1470 
1471       String pwd = ColumnImpl.decodeUncompressedText(pwdBytes, getCharset());
1472 
1473       // remove any trailing null chars
1474       int idx = pwd.indexOf('\0');
1475       if(idx >= 0) {
1476         pwd = pwd.substring(0, idx);
1477       }
1478 
1479       return pwd;
1480     } finally {
1481       releaseSharedBuffer(buffer);
1482     }
1483   }
1484 
1485   /**
1486    * Finds the relationships matching the given from and to tables from the
1487    * given cursor and adds them to the given list.
1488    */
1489   private void collectRelationships(
1490       Cursor cursor, TableImpl fromTable, TableImpl toTable,
1491       List<Relationship> relationships, boolean includeSystemTables)
1492     throws IOException
1493   {
1494     String fromTableName = ((fromTable != null) ? fromTable.getName() : null);
1495     String toTableName = ((toTable != null) ? toTable.getName() : null);
1496 
1497     for(Row row : cursor) {
1498       String fromName = row.getString(REL_COL_FROM_TABLE);
1499       String toName = row.getString(REL_COL_TO_TABLE);
1500       
1501       if(((fromTableName == null) || 
1502           fromTableName.equalsIgnoreCase(fromName)) &&
1503          ((toTableName == null) || 
1504           toTableName.equalsIgnoreCase(toName))) {
1505 
1506         String relName = row.getString(REL_COL_NAME);
1507         
1508         // found more info for a relationship.  see if we already have some
1509         // info for this relationship
1510         Relationship rel = null;
1511         for(Relationship tmp : relationships) {
1512           if(tmp.getName().equalsIgnoreCase(relName)) {
1513             rel = tmp;
1514             break;
1515           }
1516         }
1517 
1518         TableImpl relFromTable = fromTable;
1519         if(relFromTable == null) {
1520           relFromTable = getTable(fromName, includeSystemTables);
1521           if(relFromTable == null) {
1522             // invalid table or ignoring system tables, just ignore
1523             continue;
1524           }
1525         }
1526         TableImpl relToTable = toTable;
1527         if(relToTable == null) {
1528           relToTable = getTable(toName, includeSystemTables);
1529           if(relToTable == null) {
1530             // invalid table or ignoring system tables, just ignore
1531             continue;
1532           }
1533         }
1534 
1535         if(rel == null) {
1536           // new relationship
1537           int numCols = row.getInt(REL_COL_COLUMN_COUNT);
1538           int flags = row.getInt(REL_COL_FLAGS);
1539           rel = new RelationshipImpl(relName, relFromTable, relToTable,
1540                                      flags, numCols);
1541           relationships.add(rel);
1542         }
1543 
1544         // add column info
1545         int colIdx = row.getInt(REL_COL_COLUMN_INDEX);
1546         ColumnImpl fromCol = relFromTable.getColumn(
1547             row.getString(REL_COL_FROM_COLUMN));
1548         ColumnImpl toCol = relToTable.getColumn(
1549             row.getString(REL_COL_TO_COLUMN));
1550 
1551         rel.getFromColumns().set(colIdx, fromCol);
1552         rel.getToColumns().set(colIdx, toCol);
1553       }
1554     }    
1555   }
1556   
1557   /**
1558    * Add a new table to the system catalog
1559    * @param name Table name
1560    * @param objectId id of the new object
1561    */
1562   private void addToSystemCatalog(String name, int objectId, Short type, 
1563                                   String linkedDbName, String linkedTableName,
1564                                   Integer parentId)
1565     throws IOException
1566   {
1567     byte[] owner = getNewObjectOwner();
1568     Object[] catalogRow = new Object[_systemCatalog.getColumnCount()];
1569     int idx = 0;
1570     Date creationTime = new Date();
1571     for (Iterator<ColumnImpl> iter = _systemCatalog.getColumns().iterator();
1572          iter.hasNext(); idx++)
1573     {
1574       ColumnImpl col = iter.next();
1575       if (CAT_COL_ID.equals(col.getName())) {
1576         catalogRow[idx] = Integer.valueOf(objectId);
1577       } else if (CAT_COL_NAME.equals(col.getName())) {
1578         catalogRow[idx] = name;
1579       } else if (CAT_COL_TYPE.equals(col.getName())) {
1580         catalogRow[idx] = type;
1581       } else if (CAT_COL_DATE_CREATE.equals(col.getName()) ||
1582                  CAT_COL_DATE_UPDATE.equals(col.getName())) {
1583         catalogRow[idx] = creationTime;
1584       } else if (CAT_COL_PARENT_ID.equals(col.getName())) {
1585         catalogRow[idx] = parentId;
1586       } else if (CAT_COL_FLAGS.equals(col.getName())) {
1587         catalogRow[idx] = Integer.valueOf(0);
1588       } else if (CAT_COL_OWNER.equals(col.getName())) {
1589         catalogRow[idx] = owner;
1590       } else if (CAT_COL_DATABASE.equals(col.getName())) {
1591         catalogRow[idx] = linkedDbName;
1592       } else if (CAT_COL_FOREIGN_NAME.equals(col.getName())) {
1593         catalogRow[idx] = linkedTableName;
1594       }
1595     }
1596     _systemCatalog.addRow(catalogRow);
1597   }
1598 
1599   /**
1600    * Adds a new object to the system's access control entries
1601    */
1602   private void addToAccessControlEntries(
1603       Integer objectId, Integer parentId, List<byte[]> sids) 
1604     throws IOException 
1605   {
1606     if(sids.isEmpty()) {
1607       collectNewObjectSIDs(parentId, sids);
1608     }
1609 
1610     TableImpl acEntries = getAccessControlEntries();
1611     ColumnImpl acmCol = acEntries.getColumn(ACE_COL_ACM);
1612     ColumnImpl inheritCol = acEntries.getColumn(ACE_COL_F_INHERITABLE);
1613     ColumnImpl objIdCol = acEntries.getColumn(ACE_COL_OBJECT_ID);
1614     ColumnImpl sidCol = acEntries.getColumn(ACE_COL_SID);
1615 
1616     // construct a collection of ACE entries
1617     List<Object[]> aceRows = new ArrayList<Object[]>(sids.size());
1618     for(byte[] sid : sids) {
1619       Object[] aceRow = new Object[acEntries.getColumnCount()];
1620       acmCol.setRowValue(aceRow, SYS_FULL_ACCESS_ACM);
1621       inheritCol.setRowValue(aceRow, Boolean.FALSE);
1622       objIdCol.setRowValue(aceRow, objectId);
1623       sidCol.setRowValue(aceRow, sid);
1624       aceRows.add(aceRow);
1625     }
1626     acEntries.addRows(aceRows);  
1627   }
1628 
1629   /**
1630    * Find collection of SIDs for the given parent id.
1631    */
1632   private void collectNewObjectSIDs(Integer parentId, List<byte[]> sids) 
1633     throws IOException
1634   {
1635     // search for ACEs matching the given parentId.  use the index on the
1636     // objectId column if found (should be there)
1637     Cursor cursor = createCursorWithOptionalIndex(
1638         getAccessControlEntries(), ACE_COL_OBJECT_ID, parentId);
1639     
1640     for(Row row : cursor) {
1641       Integer objId = row.getInt(ACE_COL_OBJECT_ID);
1642       if(parentId.equals(objId)) {
1643         sids.add(row.getBytes(ACE_COL_SID));
1644       }
1645     }
1646 
1647     if(sids.isEmpty()) {
1648       // if all else fails, use the hard-coded default
1649       sids.add(SYS_DEFAULT_SID);
1650     }
1651   }
1652 
1653   /**
1654    * Reads a table with the given name from the given pageNumber.
1655    */
1656   private TableImpl readTable(String name, int pageNumber, int flags)
1657     throws IOException
1658   {
1659     // first, check for existing table
1660     TableImpl table = _tableCache.get(pageNumber);
1661     if(table != null) {
1662       return table;
1663     }
1664     
1665     ByteBuffer buffer = takeSharedBuffer();
1666     try {
1667       // need to load table from db
1668       _pageChannel.readPage(buffer, pageNumber);
1669       byte pageType = buffer.get(0);
1670       if (pageType != PageTypes.TABLE_DEF) {
1671         throw new IOException(withErrorContext(
1672             "Looking for " + name + " at page " + pageNumber +
1673             ", but page type is " + pageType));
1674       }
1675       return _tableCache.put(
1676           new TableImpl(this, buffer, pageNumber, name, flags));
1677     } finally {
1678       releaseSharedBuffer(buffer);
1679     }
1680   }
1681 
1682   /**
1683    * Creates a Cursor restricted to the given column value if possible (using
1684    * an existing index), otherwise a simple table cursor.
1685    */
1686   private Cursor createCursorWithOptionalIndex(
1687       TableImpl table, String colName, Object colValue)
1688     throws IOException
1689   {
1690     try {
1691       return table.newCursor()
1692         .setIndexByColumnNames(colName)
1693         .setSpecificEntry(colValue)
1694         .toCursor();
1695     } catch(IllegalArgumentException e) {
1696       if(LOG.isDebugEnabled()) {
1697         LOG.debug(withErrorContext(
1698             "Could not find expected index on table " + table.getName()));
1699       } 
1700     }
1701     // use table scan instead
1702     return CursorImpl.createCursor(table);
1703   }
1704   
1705   public void flush() throws IOException {
1706     if(_linkedDbs != null) {
1707       for(Database linkedDb : _linkedDbs.values()) {
1708         linkedDb.flush();
1709       }
1710     }
1711     _pageChannel.flush();
1712   }
1713   
1714   public void close() throws IOException {
1715     if(_linkedDbs != null) {
1716       for(Database linkedDb : _linkedDbs.values()) {
1717         linkedDb.close();
1718       }
1719     }
1720     _pageChannel.close();
1721   }
1722 
1723   public void validateNewTableName(String name) throws IOException {
1724     validateIdentifierName(name, getFormat().MAX_TABLE_NAME_LENGTH, "table");
1725 
1726     if(lookupTable(name) != null) {
1727       throw new IllegalArgumentException(withErrorContext(
1728               "Cannot create table with name of existing table '" + name + "'"));
1729     }
1730   }
1731   
1732   /**
1733    * Validates an identifier name.
1734    *
1735    * Names of fields, controls, and objects in Microsoft Access:
1736    * <ul>
1737    * <li>Can include any combination of letters, numbers, spaces, and special
1738    *     characters except a period (.), an exclamation point (!), an accent
1739    *     grave (`), and brackets ([ ]).</li>
1740    * <li>Can't begin with leading spaces.</li>
1741    * <li>Can't include control characters (ASCII values 0 through 31).</li>
1742    * </ul>
1743    * 
1744    * @usage _advanced_method_
1745    */
1746   public static void validateIdentifierName(String name,
1747                                             int maxLength,
1748                                             String identifierType)
1749   {
1750     // basic name validation
1751     validateName(name, maxLength, identifierType);
1752 
1753     // additional identifier validation
1754     if(INVALID_IDENTIFIER_CHARS.matcher(name).find()) {
1755       throw new IllegalArgumentException(
1756           identifierType + " name '" + name + "' contains invalid characters");
1757     }
1758 
1759     // cannot start with spaces
1760     if(name.charAt(0) == ' ') {
1761       throw new IllegalArgumentException(
1762           identifierType + " name '" + name +
1763           "' cannot start with a space character");
1764     }
1765   }
1766 
1767   /**
1768    * Validates a name.
1769    */
1770   private static void validateName(String name, int maxLength, String nameType)
1771   {
1772     if(isBlank(name)) {
1773       throw new IllegalArgumentException(
1774           nameType + " must have non-blank name");
1775     }
1776     if(name.length() > maxLength) {
1777       throw new IllegalArgumentException(
1778           nameType + " name is longer than max length of " + maxLength +
1779           ": " + name);
1780     }
1781   }
1782 
1783   /**
1784    * Returns {@code true} if the given string is {@code null} or all blank
1785    * space, {@code false} otherwise.
1786    */
1787   public static boolean isBlank(String name) {
1788     return((name == null) || (name.trim().length() == 0));
1789   }
1790   
1791   @Override
1792   public String toString() {
1793     return ToStringBuilder.reflectionToString(this);
1794   }
1795 
1796   /**
1797    * Adds a table to the _tableLookup and resets the _tableNames set
1798    */
1799   private void addTable(String tableName, Integer pageNumber, Short type, 
1800                         String linkedDbName, String linkedTableName)
1801   {
1802     _tableLookup.put(toLookupName(tableName),
1803                      createTableInfo(tableName, pageNumber, 0, type, 
1804                                      linkedDbName, linkedTableName));
1805     // clear this, will be created next time needed
1806     _tableNames = null;
1807   }
1808 
1809   /**
1810    * Creates a TableInfo instance appropriate for the given table data.
1811    */
1812   private static TableInfo createTableInfo(
1813       String tableName, Integer pageNumber, int flags, Short type, 
1814       String linkedDbName, String linkedTableName)
1815   {
1816     if(TYPE_LINKED_TABLE.equals(type)) {
1817       return new LinkedTableInfo(pageNumber, tableName, flags, linkedDbName,
1818                                  linkedTableName);
1819     }
1820     return new TableInfo(pageNumber, tableName, flags);
1821   }
1822 
1823   /**
1824    * @return the tableInfo of the given table, if any
1825    */
1826   private TableInfo lookupTable(String tableName) throws IOException {
1827 
1828     String lookupTableName = toLookupName(tableName);
1829     TableInfo tableInfo = _tableLookup.get(lookupTableName);
1830     if(tableInfo != null) {
1831       return tableInfo;
1832     }
1833 
1834     tableInfo = _tableFinder.lookupTable(tableName);
1835 
1836     if(tableInfo != null) {
1837       // cache for later
1838       _tableLookup.put(lookupTableName, tableInfo);
1839     }
1840 
1841     return tableInfo;
1842   }
1843 
1844   /**
1845    * @return a string usable in the _tableLookup map.
1846    */
1847   public static String toLookupName(String name) {
1848     return ((name != null) ? name.toUpperCase() : null);
1849   }
1850 
1851   /**
1852    * @return {@code true} if the given flags indicate that an object is some
1853    *         sort of system object, {@code false} otherwise.
1854    */
1855   private static boolean isSystemObject(int flags) {
1856     return ((flags & SYSTEM_OBJECT_FLAGS) != 0);
1857   }
1858 
1859   /**
1860    * Returns the default TimeZone.  This is normally the platform default
1861    * TimeZone as returned by {@link TimeZone#getDefault}, but can be
1862    * overridden using the system property
1863    * {@value com.healthmarketscience.jackcess.Database#TIMEZONE_PROPERTY}.
1864    * @usage _advanced_method_
1865    */
1866   public static TimeZone getDefaultTimeZone()
1867   {
1868     String tzProp = System.getProperty(TIMEZONE_PROPERTY);
1869     if(tzProp != null) {
1870       tzProp = tzProp.trim();
1871       if(tzProp.length() > 0) {
1872         return TimeZone.getTimeZone(tzProp);
1873       }
1874     }
1875 
1876     // use system default
1877     return TimeZone.getDefault();
1878   }
1879   
1880   /**
1881    * Returns the default Charset for the given JetFormat.  This may or may not
1882    * be platform specific, depending on the format, but can be overridden
1883    * using a system property composed of the prefix
1884    * {@value com.healthmarketscience.jackcess.Database#CHARSET_PROPERTY_PREFIX}
1885    * followed by the JetFormat version to which the charset should apply,
1886    * e.g. {@code "com.healthmarketscience.jackcess.charset.VERSION_3"}.
1887    * @usage _advanced_method_
1888    */
1889   public static Charset getDefaultCharset(JetFormat format)
1890   {
1891     String csProp = System.getProperty(CHARSET_PROPERTY_PREFIX + format);
1892     if(csProp != null) {
1893       csProp = csProp.trim();
1894       if(csProp.length() > 0) {
1895         return Charset.forName(csProp);
1896       }
1897     }
1898 
1899     // use format default
1900     return format.CHARSET;
1901   }
1902   
1903   /**
1904    * Returns the default Table.ColumnOrder.  This defaults to
1905    * {@link Database#DEFAULT_COLUMN_ORDER}, but can be overridden using the system
1906    * property {@value com.healthmarketscience.jackcess.Database#COLUMN_ORDER_PROPERTY}.
1907    * @usage _advanced_method_
1908    */
1909   public static Table.ColumnOrder getDefaultColumnOrder()
1910   {
1911     String coProp = System.getProperty(COLUMN_ORDER_PROPERTY);
1912     if(coProp != null) {
1913       coProp = coProp.trim();
1914       if(coProp.length() > 0) {
1915         return Table.ColumnOrder.valueOf(coProp);
1916       }
1917     }
1918 
1919     // use default order
1920     return DEFAULT_COLUMN_ORDER;
1921   }
1922   
1923   /**
1924    * Returns the default enforce foreign-keys policy.  This defaults to
1925    * {@code true}, but can be overridden using the system
1926    * property {@value com.healthmarketscience.jackcess.Database#FK_ENFORCE_PROPERTY}.
1927    * @usage _advanced_method_
1928    */
1929   public static boolean getDefaultEnforceForeignKeys()
1930   {
1931     String prop = System.getProperty(FK_ENFORCE_PROPERTY);
1932     if(prop != null) {
1933       return Boolean.TRUE.toString().equalsIgnoreCase(prop);
1934     }
1935     return true;
1936   }
1937   
1938   /**
1939    * Returns the default allow auto number insert policy.  This defaults to
1940    * {@code false}, but can be overridden using the system
1941    * property {@value com.healthmarketscience.jackcess.Database#ALLOW_AUTONUM_INSERT_PROPERTY}.
1942    * @usage _advanced_method_
1943    */
1944   public static boolean getDefaultAllowAutoNumberInsert()
1945   {
1946     String prop = System.getProperty(ALLOW_AUTONUM_INSERT_PROPERTY);
1947     if(prop != null) {
1948       return Boolean.TRUE.toString().equalsIgnoreCase(prop);
1949     }
1950     return false;
1951   }
1952   
1953   /**
1954    * Copies the given db InputStream to the given channel using the most
1955    * efficient means possible.
1956    */
1957   protected static void transferDbFrom(FileChannel channel, InputStream in)
1958     throws IOException
1959   {
1960     ReadableByteChannel readChannel = Channels.newChannel(in);
1961     if(!BROKEN_NIO) {
1962       // sane implementation
1963       channel.transferFrom(readChannel, 0, MAX_EMPTYDB_SIZE);    
1964     } else {
1965       // do things the hard way for broken vms
1966       ByteBuffer bb = ByteBuffer.allocate(8096);
1967       while(readChannel.read(bb) >= 0) {
1968         bb.flip();
1969         channel.write(bb);
1970         bb.clear();
1971       }
1972     }
1973   }
1974 
1975   /**
1976    * Returns the password mask retrieved from the given header page and
1977    * format, or {@code null} if this format does not use a password mask.
1978    */
1979   static byte[] getPasswordMask(ByteBuffer buffer, JetFormat format)
1980   {
1981     // get extra password mask if necessary (the extra password mask is
1982     // generated from the database creation date stored in the header)
1983     int pwdMaskPos = format.OFFSET_HEADER_DATE;
1984     if(pwdMaskPos < 0) {
1985       return null;
1986     }
1987 
1988     buffer.position(pwdMaskPos);
1989     double dateVal = Double.longBitsToDouble(buffer.getLong());
1990 
1991     byte[] pwdMask = new byte[4];
1992     PageChannel.wrap(pwdMask).putInt((int)dateVal);
1993 
1994     return pwdMask;
1995   }
1996 
1997   protected static InputStream getResourceAsStream(String resourceName)
1998     throws IOException
1999   {
2000     InputStream stream = DatabaseImpl.class.getClassLoader()
2001       .getResourceAsStream(resourceName);
2002     
2003     if(stream == null) {
2004       
2005       stream = Thread.currentThread().getContextClassLoader()
2006         .getResourceAsStream(resourceName);
2007       
2008       if(stream == null) {
2009         throw new IOException("Could not load jackcess resource " +
2010                               resourceName);
2011       }
2012     }
2013 
2014     return stream;
2015   }
2016 
2017   private static boolean isTableType(Short objType) {
2018     return(TYPE_TABLE.equals(objType) || TYPE_LINKED_TABLE.equals(objType));
2019   }
2020 
2021   public static FileFormatDetails getFileFormatDetails(FileFormat fileFormat) {
2022     return FILE_FORMAT_DETAILS.get(fileFormat);
2023   }
2024 
2025   private static void addFileFormatDetails(
2026       FileFormat fileFormat, String emptyFileName, JetFormat format)
2027   {
2028     String emptyFile = 
2029       ((emptyFileName != null) ? 
2030        RESOURCE_PATH + emptyFileName + fileFormat.getFileExtension() : null);
2031     FILE_FORMAT_DETAILS.put(fileFormat, new FileFormatDetails(emptyFile, format));
2032   }
2033 
2034   private static String getName(File file) {
2035     if(file == null) {
2036       return "<UNKNOWN.DB>";
2037     } 
2038     return file.getName();
2039   }
2040 
2041   private String withErrorContext(String msg) {
2042     return withErrorContext(msg, getName());
2043   }
2044 
2045   private static String withErrorContext(String msg, String dbName) {
2046     return msg + " (Db=" + dbName + ")";
2047   }
2048 
2049   /**
2050    * Utility class for storing table page number and actual name.
2051    */
2052   private static class TableInfo implements TableMetaData
2053   {
2054     public final Integer pageNumber;
2055     public final String tableName;
2056     public final int flags;
2057 
2058     private TableInfo(Integer newPageNumber, String newTableName, int newFlags) {
2059       pageNumber = newPageNumber;
2060       tableName = newTableName;
2061       flags = newFlags;
2062     }
2063 
2064     public String getName() {
2065       return tableName;
2066     }
2067     
2068     public boolean isLinked() {
2069       return false;
2070     }
2071 
2072     public boolean isSystem() {
2073       return isSystemObject(flags);
2074     } 
2075 
2076     public String getLinkedTableName() {
2077       return null;
2078     }
2079 
2080     public String getLinkedDbName() {
2081       return null;
2082     }
2083 
2084     public Table open(Database db) throws IOException {
2085       return ((DatabaseImpl)db).getTable(this, true);
2086     }
2087 
2088     @Override
2089     public String toString() {
2090       ToStringBuilder sb = CustomToStringStyle.valueBuilder("TableMetaData")
2091         .append("name", getName());
2092         if(isSystem()) {
2093           sb.append("isSystem", isSystem());
2094         }
2095         if(isLinked()) {
2096           sb.append("isLinked", isLinked())
2097             .append("linkedTableName", getLinkedTableName())
2098             .append("linkedDbName", getLinkedDbName());
2099         }
2100         return sb.toString();
2101     }
2102   }
2103 
2104   /**
2105    * Utility class for storing linked table info
2106    */
2107   private static class LinkedTableInfo extends TableInfo
2108   {
2109     private final String linkedDbName;
2110     private final String linkedTableName;
2111 
2112     private LinkedTableInfo(Integer newPageNumber, String newTableName, 
2113                             int newFlags, String newLinkedDbName, 
2114                             String newLinkedTableName) {
2115       super(newPageNumber, newTableName, newFlags);
2116       linkedDbName = newLinkedDbName;
2117       linkedTableName = newLinkedTableName;
2118     }
2119 
2120     @Override
2121     public boolean isLinked() {
2122       return true;
2123     }
2124 
2125     @Override
2126     public String getLinkedTableName() {
2127       return linkedTableName;
2128     }
2129 
2130     @Override
2131     public String getLinkedDbName() {
2132       return linkedDbName;
2133     }
2134   }
2135 
2136   /**
2137    * Table iterator for this database, unmodifiable.
2138    */
2139   private class TableIterator implements Iterator<Table>
2140   {
2141     private Iterator<String> _tableNameIter;
2142 
2143     private TableIterator(Set<String> tableNames) {
2144       _tableNameIter = tableNames.iterator();
2145     }
2146 
2147     public boolean hasNext() {
2148       return _tableNameIter.hasNext();
2149     }
2150 
2151     public void remove() {
2152       throw new UnsupportedOperationException();
2153     }
2154 
2155     public Table next() {
2156       if(!hasNext()) {
2157         throw new NoSuchElementException();
2158       }
2159       try {
2160         return getTable(_tableNameIter.next(), true);
2161       } catch(IOException e) {
2162         throw new RuntimeIOException(e);
2163       }
2164     }
2165   }
2166 
2167   /**
2168    * Utility class for handling table lookups.
2169    */
2170   private abstract class TableFinder
2171   {
2172     public Integer findObjectId(Integer parentId, String name) 
2173       throws IOException 
2174     {
2175       Cursor cur = findRow(parentId, name);
2176       if(cur == null) {  
2177         return null;
2178       }
2179       ColumnImpl idCol = _systemCatalog.getColumn(CAT_COL_ID);
2180       return (Integer)cur.getCurrentRowValue(idCol);
2181     }
2182 
2183     public Row getObjectRow(Integer parentId, String name,
2184                             Collection<String> columns) 
2185       throws IOException 
2186     {
2187       Cursor cur = findRow(parentId, name);
2188       return ((cur != null) ? cur.getCurrentRow(columns) : null);
2189     }
2190 
2191     public Row getObjectRow(
2192         Integer objectId, Collection<String> columns)
2193       throws IOException
2194     {
2195       Cursor cur = findRow(objectId);
2196       return ((cur != null) ? cur.getCurrentRow(columns) : null);
2197     }
2198 
2199     public void getTableNames(Set<String> tableNames,
2200                               boolean normalTables,
2201                               boolean systemTables,
2202                               boolean linkedTables)
2203       throws IOException
2204     {
2205       for(Row row : getTableNamesCursor().newIterable().setColumnNames(
2206               SYSTEM_CATALOG_COLUMNS)) {
2207 
2208         String tableName = row.getString(CAT_COL_NAME);
2209         int flags = row.getInt(CAT_COL_FLAGS);
2210         Short type = row.getShort(CAT_COL_TYPE);
2211         int parentId = row.getInt(CAT_COL_PARENT_ID);
2212 
2213         if(parentId != _tableParentId) {
2214           continue;
2215         }
2216 
2217         if(TYPE_TABLE.equals(type)) {
2218           if(!isSystemObject(flags)) {
2219             if(normalTables) {
2220               tableNames.add(tableName);
2221             }
2222           } else if(systemTables) {
2223             tableNames.add(tableName);
2224           }
2225         } else if(TYPE_LINKED_TABLE.equals(type) && linkedTables) {
2226           tableNames.add(tableName);
2227         }
2228       }
2229     }
2230 
2231     public boolean isLinkedTable(Table table) throws IOException
2232     {
2233       for(Row row : getTableNamesCursor().newIterable().setColumnNames(
2234               SYSTEM_CATALOG_TABLE_DETAIL_COLUMNS)) {
2235         Short type = row.getShort(CAT_COL_TYPE);
2236         String linkedDbName = row.getString(CAT_COL_DATABASE);
2237         String linkedTableName = row.getString(CAT_COL_FOREIGN_NAME);
2238 
2239         if(TYPE_LINKED_TABLE.equals(type) &&
2240            matchesLinkedTable(table, linkedTableName, linkedDbName)) {
2241           return true;
2242         } 
2243       }
2244       return false;
2245     }
2246 
2247     protected abstract Cursor findRow(Integer parentId, String name)
2248       throws IOException;
2249 
2250     protected abstract Cursor findRow(Integer objectId) 
2251       throws IOException;
2252 
2253     protected abstract Cursor getTableNamesCursor() throws IOException;
2254 
2255     public abstract TableInfo lookupTable(String tableName)
2256       throws IOException;
2257 
2258     protected abstract int findMaxSyntheticId() throws IOException;
2259 
2260     public int getNextFreeSyntheticId() throws IOException
2261     {
2262       int maxSynthId = findMaxSyntheticId();
2263       if(maxSynthId >= -1) {
2264         // bummer, no more ids available
2265         throw new IllegalStateException(withErrorContext(
2266                 "Too many database objects!"));
2267       }
2268       return maxSynthId + 1;
2269     }
2270   }
2271 
2272   /**
2273    * Normal table lookup handler, using catalog table index.
2274    */
2275   private final class DefaultTableFinder extends TableFinder
2276   {
2277     private final IndexCursor _systemCatalogCursor;
2278     private IndexCursor _systemCatalogIdCursor;
2279 
2280     private DefaultTableFinder(IndexCursor systemCatalogCursor) {
2281       _systemCatalogCursor = systemCatalogCursor;
2282     }
2283     
2284     private void initIdCursor() throws IOException {
2285       if(_systemCatalogIdCursor == null) {
2286         _systemCatalogIdCursor = _systemCatalog.newCursor()
2287           .setIndexByColumnNames(CAT_COL_ID)
2288           .toIndexCursor();
2289       }
2290     }
2291 
2292     @Override
2293     protected Cursor findRow(Integer parentId, String name) 
2294       throws IOException 
2295     {
2296       return (_systemCatalogCursor.findFirstRowByEntry(parentId, name) ?
2297               _systemCatalogCursor : null);
2298     }
2299 
2300     @Override
2301     protected Cursor findRow(Integer objectId) throws IOException 
2302     {
2303       initIdCursor();
2304       return (_systemCatalogIdCursor.findFirstRowByEntry(objectId) ?
2305               _systemCatalogIdCursor : null);
2306     }
2307 
2308     @Override
2309     public TableInfo lookupTable(String tableName) throws IOException {
2310 
2311       if(findRow(_tableParentId, tableName) == null) {
2312         return null;
2313       }
2314 
2315       Row row = _systemCatalogCursor.getCurrentRow(
2316           SYSTEM_CATALOG_TABLE_DETAIL_COLUMNS);
2317       Integer pageNumber = row.getInt(CAT_COL_ID);
2318       String realName = row.getString(CAT_COL_NAME);
2319       int flags = row.getInt(CAT_COL_FLAGS);
2320       Short type = row.getShort(CAT_COL_TYPE);
2321 
2322       if(!isTableType(type)) {
2323         return null;
2324       }
2325 
2326       String linkedDbName = row.getString(CAT_COL_DATABASE);
2327       String linkedTableName = row.getString(CAT_COL_FOREIGN_NAME);
2328 
2329       return createTableInfo(realName, pageNumber, flags, type, linkedDbName,
2330                              linkedTableName);
2331     }
2332     
2333     @Override
2334     protected Cursor getTableNamesCursor() throws IOException {
2335       return _systemCatalogCursor.getIndex().newCursor()
2336         .setStartEntry(_tableParentId, IndexData.MIN_VALUE)
2337         .setEndEntry(_tableParentId, IndexData.MAX_VALUE)
2338         .toIndexCursor();
2339     }
2340 
2341     @Override
2342     protected int findMaxSyntheticId() throws IOException {
2343       initIdCursor();
2344       _systemCatalogIdCursor.reset();
2345 
2346       // synthetic ids count up from min integer.  so the current, highest,
2347       // in-use synthetic id is the max id < 0.
2348       _systemCatalogIdCursor.findClosestRowByEntry(0);
2349       if(!_systemCatalogIdCursor.moveToPreviousRow()) {
2350         return Integer.MIN_VALUE;
2351       }
2352       ColumnImpl idCol = _systemCatalog.getColumn(CAT_COL_ID);
2353       return (Integer)_systemCatalogIdCursor.getCurrentRowValue(idCol);
2354     }
2355   }
2356   
2357   /**
2358    * Fallback table lookup handler, using catalog table scans.
2359    */
2360   private final class FallbackTableFinder extends TableFinder
2361   {
2362     private final Cursor _systemCatalogCursor;
2363 
2364     private FallbackTableFinder(Cursor systemCatalogCursor) {
2365       _systemCatalogCursor = systemCatalogCursor;
2366     }
2367 
2368     @Override
2369     protected Cursor findRow(Integer parentId, String name) 
2370       throws IOException 
2371     {
2372       Map<String,Object> rowPat = new HashMap<String,Object>();
2373       rowPat.put(CAT_COL_PARENT_ID, parentId);  
2374       rowPat.put(CAT_COL_NAME, name);
2375       return (_systemCatalogCursor.findFirstRow(rowPat) ?
2376               _systemCatalogCursor : null);
2377     }
2378 
2379     @Override
2380     protected Cursor findRow(Integer objectId) throws IOException 
2381     {
2382       ColumnImpl idCol = _systemCatalog.getColumn(CAT_COL_ID);
2383       return (_systemCatalogCursor.findFirstRow(idCol, objectId) ?
2384               _systemCatalogCursor : null);
2385     }
2386 
2387     @Override
2388     public TableInfo lookupTable(String tableName) throws IOException {
2389 
2390       for(Row row : _systemCatalogCursor.newIterable().setColumnNames(
2391               SYSTEM_CATALOG_TABLE_DETAIL_COLUMNS)) {
2392 
2393         Short type = row.getShort(CAT_COL_TYPE);
2394         if(!isTableType(type)) {
2395           continue;
2396         }
2397 
2398         int parentId = row.getInt(CAT_COL_PARENT_ID);
2399         if(parentId != _tableParentId) {
2400           continue;
2401         }
2402 
2403         String realName = row.getString(CAT_COL_NAME);
2404         if(!tableName.equalsIgnoreCase(realName)) {
2405           continue;
2406         }
2407 
2408         Integer pageNumber = row.getInt(CAT_COL_ID);
2409         int flags = row.getInt(CAT_COL_FLAGS);
2410         String linkedDbName = row.getString(CAT_COL_DATABASE);
2411         String linkedTableName = row.getString(CAT_COL_FOREIGN_NAME);
2412 
2413         return createTableInfo(realName, pageNumber, flags, type, linkedDbName,
2414                                linkedTableName);
2415       }
2416 
2417       return null;
2418     }
2419     
2420     @Override
2421     protected Cursor getTableNamesCursor() throws IOException {
2422       return _systemCatalogCursor;
2423     }
2424 
2425     @Override
2426     protected int findMaxSyntheticId() throws IOException {
2427       // find max id < 0
2428       ColumnImpl idCol = _systemCatalog.getColumn(CAT_COL_ID);
2429       _systemCatalogCursor.reset();
2430       int curMaxSynthId = Integer.MIN_VALUE;
2431       while(_systemCatalogCursor.moveToNextRow()) {
2432         int id = (Integer)_systemCatalogCursor.getCurrentRowValue(idCol);
2433         if((id > curMaxSynthId) && (id < 0)) {
2434           curMaxSynthId = id;
2435         }
2436       }
2437       return curMaxSynthId;
2438     }
2439   }
2440 
2441   /**
2442    * WeakReference for a Table which holds the table pageNumber (for later
2443    * cache purging).
2444    */
2445   private static final class WeakTableReference extends WeakReference<TableImpl>
2446   {
2447     private final Integer _pageNumber;
2448 
2449     private WeakTableReference(Integer pageNumber, TableImpl table, 
2450                                ReferenceQueue<TableImpl> queue) {
2451       super(table, queue);
2452       _pageNumber = pageNumber;
2453     }
2454 
2455     public Integer getPageNumber() {
2456       return _pageNumber;
2457     }
2458   }
2459 
2460   /**
2461    * Cache of currently in-use tables, allows re-use of existing tables.
2462    */
2463   private static final class TableCache
2464   {
2465     private final Map<Integer,WeakTableReference> _tables = 
2466       new HashMap<Integer,WeakTableReference>();
2467     private final ReferenceQueue<TableImpl> _queue = 
2468       new ReferenceQueue<TableImpl>();
2469 
2470     public TableImpl get(Integer pageNumber) {
2471       WeakTableReference ref = _tables.get(pageNumber);
2472       return ((ref != null) ? ref.get() : null);
2473     }
2474 
2475     public TableImpl put(TableImpl table) {
2476       purgeOldRefs();
2477   
2478       Integer pageNumber = table.getTableDefPageNumber();
2479       WeakTableReference ref = new WeakTableReference(
2480           pageNumber, table, _queue);
2481       _tables.put(pageNumber, ref);
2482 
2483       return table;
2484     }
2485 
2486     private void purgeOldRefs() {
2487       WeakTableReference oldRef = null;
2488       while((oldRef = (WeakTableReference)_queue.poll()) != null) {
2489         _tables.remove(oldRef.getPageNumber());
2490       }
2491     }
2492   }
2493 
2494   /**
2495    * Internal details for each FileForrmat
2496    * @usage _advanced_class_
2497    */
2498   public static final class FileFormatDetails
2499   {
2500     private final String _emptyFile;
2501     private final JetFormat _format;
2502 
2503     private FileFormatDetails(String emptyFile, JetFormat format) {
2504       _emptyFile = emptyFile;
2505       _format = format;
2506     }
2507 
2508     public String getEmptyFilePath() {
2509       return _emptyFile;
2510     }
2511 
2512     public JetFormat getFormat() {
2513       return _format;
2514     }
2515   }
2516 }