View Javadoc
1   /*
2   Copyright (c) 2005 Health Market Science, Inc.
3   
4   Licensed under the Apache License, Version 2.0 (the "License");
5   you may not use this file except in compliance with the License.
6   You may obtain a copy of the License at
7   
8       http://www.apache.org/licenses/LICENSE-2.0
9   
10  Unless required by applicable law or agreed to in writing, software
11  distributed under the License is distributed on an "AS IS" BASIS,
12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  See the License for the specific language governing permissions and
14  limitations under the License.
15  */
16  
17  package com.healthmarketscience.jackcess.impl;
18  
19  import java.io.BufferedWriter;
20  import java.io.IOException;
21  import java.io.StringWriter;
22  import java.nio.BufferOverflowException;
23  import java.nio.ByteBuffer;
24  import java.nio.charset.Charset;
25  import java.util.AbstractMap;
26  import java.util.ArrayList;
27  import java.util.Arrays;
28  import java.util.Collection;
29  import java.util.Collections;
30  import java.util.Comparator;
31  import java.util.HashMap;
32  import java.util.Iterator;
33  import java.util.LinkedHashSet;
34  import java.util.List;
35  import java.util.Map;
36  import java.util.Set;
37  import java.util.TreeSet;
38  
39  import com.healthmarketscience.jackcess.BatchUpdateException;
40  import com.healthmarketscience.jackcess.Column;
41  import com.healthmarketscience.jackcess.ColumnBuilder;
42  import com.healthmarketscience.jackcess.ConstraintViolationException;
43  import com.healthmarketscience.jackcess.CursorBuilder;
44  import com.healthmarketscience.jackcess.Index;
45  import com.healthmarketscience.jackcess.IndexBuilder;
46  import com.healthmarketscience.jackcess.JackcessException;
47  import com.healthmarketscience.jackcess.PropertyMap;
48  import com.healthmarketscience.jackcess.Row;
49  import com.healthmarketscience.jackcess.RowId;
50  import com.healthmarketscience.jackcess.Table;
51  import com.healthmarketscience.jackcess.util.ErrorHandler;
52  import com.healthmarketscience.jackcess.util.ExportUtil;
53  import org.apache.commons.logging.Log;
54  import org.apache.commons.logging.LogFactory;
55  
56  /**
57   * A single database table
58   * <p>
59   * Is not thread-safe.
60   * 
61   * @author Tim McCune
62   * @usage _intermediate_class_
63   */
64  public class TableImpl implements Table
65  {  
66    private static final Log LOG = LogFactory.getLog(TableImpl.class);
67  
68    private static final short OFFSET_MASK = (short)0x1FFF;
69  
70    private static final short DELETED_ROW_MASK = (short)0x8000;
71    
72    private static final short OVERFLOW_ROW_MASK = (short)0x4000;
73  
74    static final int MAGIC_TABLE_NUMBER = 1625;
75  
76    private static final int MAX_BYTE = 256;
77  
78    /**
79     * Table type code for system tables
80     * @usage _intermediate_class_
81     */
82    public static final byte TYPE_SYSTEM = 0x53;
83    /**
84     * Table type code for user tables
85     * @usage _intermediate_class_
86     */
87    public static final byte TYPE_USER = 0x4e;
88  
89    public enum IndexFeature {
90      EXACT_MATCH, EXACT_UNIQUE_ONLY, ANY_MATCH;
91    }
92  
93    /** comparator which sorts variable length columns based on their index into
94        the variable length offset table */
95    private static final Comparator<ColumnImpl> VAR_LEN_COLUMN_COMPARATOR =
96      new Comparator<ColumnImpl>() {
97        public int compare(ColumnImpl c1, ColumnImpl c2) {
98          return ((c1.getVarLenTableIndex() < c2.getVarLenTableIndex()) ? -1 :
99                  ((c1.getVarLenTableIndex() > c2.getVarLenTableIndex()) ? 1 :
100                  0));
101       }
102     };
103 
104   /** comparator which sorts columns based on their display index */
105   private static final Comparator<ColumnImpl> DISPLAY_ORDER_COMPARATOR =
106     new Comparator<ColumnImpl>() {
107       public int compare(ColumnImpl c1, ColumnImpl c2) {
108         return ((c1.getDisplayIndex() < c2.getDisplayIndex()) ? -1 :
109                 ((c1.getDisplayIndex() > c2.getDisplayIndex()) ? 1 :
110                  0));
111       }
112     };
113 
114   /** owning database */
115   private final DatabaseImpl _database;
116   /** additional table flags from the catalog entry */
117   private final int _flags;
118   /** Type of the table (either TYPE_SYSTEM or TYPE_USER) */
119   private final byte _tableType;
120   /** Number of actual indexes on the table */
121   private int _indexCount;
122   /** Number of logical indexes for the table */
123   private int _logicalIndexCount;
124   /** page number of the definition of this table */
125   private final int _tableDefPageNumber;
126   /** max Number of columns in the table (includes previous deletions) */
127   private short _maxColumnCount;
128   /** max Number of variable columns in the table */
129   private short _maxVarColumnCount;
130   /** List of columns in this table, ordered by column number */
131   private final List<ColumnImpl> _columns = new ArrayList<ColumnImpl>();
132   /** List of variable length columns in this table, ordered by offset */
133   private final List<ColumnImpl> _varColumns = new ArrayList<ColumnImpl>();
134   /** List of autonumber columns in this table, ordered by column number */
135   private final List<ColumnImpl> _autoNumColumns = new ArrayList<ColumnImpl>(1);
136   /** List of indexes on this table (multiple logical indexes may be backed by
137       the same index data) */
138   private final List<IndexImpl> _indexes = new ArrayList<IndexImpl>();
139   /** List of index datas on this table (the actual backing data for an
140       index) */
141   private final List<IndexData> _indexDatas = new ArrayList<IndexData>();
142   /** List of columns in this table which are in one or more indexes */
143   private final Set<ColumnImpl> _indexColumns = new LinkedHashSet<ColumnImpl>();
144   /** Table name as stored in Database */
145   private final String _name;
146   /** Usage map of pages that this table owns */
147   private final UsageMap _ownedPages;
148   /** Usage map of pages that this table owns with free space on them */
149   private final UsageMap _freeSpacePages;
150   /** Number of rows in the table */
151   private int _rowCount;
152   /** last long auto number for the table */
153   private int _lastLongAutoNumber;
154   /** last complex type auto number for the table */
155   private int _lastComplexTypeAutoNumber;
156   /** modification count for the table, keeps row-states up-to-date */
157   private int _modCount;
158   /** page buffer used to update data pages when adding rows */
159   private final TempPageHolder _addRowBufferH =
160     TempPageHolder.newHolder(TempBufferHolder.Type.SOFT);
161   /** page buffer used to update the table def page */
162   private final TempPageHolder _tableDefBufferH =
163     TempPageHolder.newHolder(TempBufferHolder.Type.SOFT);
164   /** buffer used to writing rows of data */
165   private final TempBufferHolder _writeRowBufferH =
166     TempBufferHolder.newHolder(TempBufferHolder.Type.SOFT, true);
167   /** page buffer used to write out-of-row "long value" data */
168   private final TempPageHolder _longValueBufferH =
169     TempPageHolder.newHolder(TempBufferHolder.Type.SOFT);
170   /** optional error handler to use when row errors are encountered */
171   private ErrorHandler _tableErrorHandler;
172   /** properties for this table */
173   private PropertyMap _props;
174   /** properties group for this table (and columns) */
175   private PropertyMaps _propertyMaps;
176   /** optional flag indicating whether or not auto numbers can be directly
177       inserted by the user */
178   private Boolean _allowAutoNumInsert;
179   /** foreign-key enforcer for this table */
180   private final FKEnforcer _fkEnforcer;
181 
182   /** default cursor for iterating through the table, kept here for basic
183       table traversal */
184   private CursorImpl _defaultCursor;
185   
186   /**
187    * Only used by unit tests
188    * @usage _advanced_method_
189    */
190   protected TableImpl(boolean testing, List<ColumnImpl> columns) 
191     throws IOException 
192   {
193     if(!testing) {
194       throw new IllegalArgumentException();
195     }
196     _database = null;
197     _tableDefPageNumber = PageChannel.INVALID_PAGE_NUMBER;
198     _name = null;
199 
200     _columns.addAll(columns);
201     for(ColumnImpl col : _columns) {
202       if(col.getType().isVariableLength()) {
203         _varColumns.add(col);
204       }
205     }
206     _maxColumnCount = (short)_columns.size();
207     _maxVarColumnCount = (short)_varColumns.size();
208     initAutoNumberColumns();
209 
210     _fkEnforcer = null;
211     _flags = 0;
212     _tableType = TYPE_USER;
213     _indexCount = 0;
214     _logicalIndexCount = 0;
215     _ownedPages = null;
216     _freeSpacePages = null;
217   }
218   
219   /**
220    * @param database database which owns this table
221    * @param tableBuffer Buffer to read the table with
222    * @param pageNumber Page number of the table definition
223    * @param name Table name
224    */
225   protected TableImpl(DatabaseImpl database, ByteBuffer tableBuffer,
226                       int pageNumber, String name, int flags)
227     throws IOException
228   {
229     _database = database;
230     _tableDefPageNumber = pageNumber;
231     _name = name;
232     _flags = flags;
233 
234     // read table definition
235     tableBuffer = loadCompleteTableDefinitionBuffer(tableBuffer, null);
236 
237     _rowCount = tableBuffer.getInt(getFormat().OFFSET_NUM_ROWS);
238     _lastLongAutoNumber = tableBuffer.getInt(getFormat().OFFSET_NEXT_AUTO_NUMBER);
239     if(getFormat().OFFSET_NEXT_COMPLEX_AUTO_NUMBER >= 0) {
240       _lastComplexTypeAutoNumber = tableBuffer.getInt(
241           getFormat().OFFSET_NEXT_COMPLEX_AUTO_NUMBER);
242     }
243     _tableType = tableBuffer.get(getFormat().OFFSET_TABLE_TYPE);
244     _maxColumnCount = tableBuffer.getShort(getFormat().OFFSET_MAX_COLS);
245     _maxVarColumnCount = tableBuffer.getShort(getFormat().OFFSET_NUM_VAR_COLS);
246     short columnCount = tableBuffer.getShort(getFormat().OFFSET_NUM_COLS);
247     _logicalIndexCount = tableBuffer.getInt(getFormat().OFFSET_NUM_INDEX_SLOTS);
248     _indexCount = tableBuffer.getInt(getFormat().OFFSET_NUM_INDEXES);
249 
250     tableBuffer.position(getFormat().OFFSET_OWNED_PAGES);
251     _ownedPages = UsageMap.read(getDatabase(), tableBuffer);
252     tableBuffer.position(getFormat().OFFSET_FREE_SPACE_PAGES);
253     _freeSpacePages = UsageMap.read(getDatabase(), tableBuffer);
254     
255     for (int i = 0; i < _indexCount; i++) {
256       _indexDatas.add(IndexData.create(this, tableBuffer, i, getFormat()));
257     }
258     
259     readColumnDefinitions(tableBuffer, columnCount);
260 
261     readIndexDefinitions(tableBuffer);
262 
263     // read column usage map info
264     while((tableBuffer.remaining() >= 2) && 
265           readColumnUsageMaps(tableBuffer)) {
266       // keep reading ...
267     }
268 
269     // re-sort columns if necessary
270     if(getDatabase().getColumnOrder() != ColumnOrder.DATA) {
271       Collections.sort(_columns, DISPLAY_ORDER_COMPARATOR);
272     }
273 
274     for(ColumnImpl col : _columns) {
275       // some columns need to do extra work after the table is completely
276       // loaded
277       col.postTableLoadInit();
278     }
279 
280     _fkEnforcer = new FKEnforcer(this);
281 
282     if(!isSystem()) {
283       // after fully constructed, allow column validator to be configured (but
284       // only for user tables)
285       for(ColumnImpl col : _columns) {
286         col.setColumnValidator(null);
287       }
288     }
289   }
290 
291   public String getName() {
292     return _name;
293   }
294 
295   public boolean isHidden() {
296     return((_flags & DatabaseImpl.HIDDEN_OBJECT_FLAG) != 0);
297   }
298 
299   public boolean isSystem() {
300     return(_tableType != TYPE_USER);
301   }
302 
303   /**
304    * @usage _advanced_method_
305    */
306   public int getMaxColumnCount() {
307     return _maxColumnCount;
308   }
309   
310   public int getColumnCount() {
311     return _columns.size();
312   }
313   
314   public DatabaseImpl getDatabase() {
315     return _database;
316   }
317   
318   /**
319    * @usage _advanced_method_
320    */
321   public JetFormat getFormat() {
322     return getDatabase().getFormat();
323   }
324 
325   /**
326    * @usage _advanced_method_
327    */
328   public PageChannel getPageChannel() {
329     return getDatabase().getPageChannel();
330   }
331 
332   public ErrorHandler getErrorHandler() {
333     return((_tableErrorHandler != null) ? _tableErrorHandler :
334            getDatabase().getErrorHandler());
335   }
336 
337   public void setErrorHandler(ErrorHandler newErrorHandler) {
338     _tableErrorHandler = newErrorHandler;
339   }    
340 
341   public int getTableDefPageNumber() {
342     return _tableDefPageNumber;
343   }
344 
345   public boolean isAllowAutoNumberInsert() {
346     return ((_allowAutoNumInsert != null) ? (boolean)_allowAutoNumInsert :
347             getDatabase().isAllowAutoNumberInsert());
348   }
349 
350   public void setAllowAutoNumberInsert(Boolean allowAutoNumInsert) {
351     _allowAutoNumInsert = allowAutoNumInsert;
352   }
353 
354   /**
355    * @usage _advanced_method_
356    */
357   public RowState createRowState() {
358     return new RowState(TempBufferHolder.Type.HARD);
359   }
360 
361   /**
362    * @usage _advanced_method_
363    */
364   public UsageMap.PageCursor getOwnedPagesCursor() {
365     return _ownedPages.cursor();
366   }
367 
368   /**
369    * Returns the <i>approximate</i> number of database pages owned by this
370    * table and all related indexes (this number does <i>not</i> take into
371    * account pages used for large OLE/MEMO fields).
372    * <p>
373    * To calculate the approximate number of bytes owned by a table:
374    * <code>
375    * int approxTableBytes = (table.getApproximateOwnedPageCount() *
376    *                         table.getFormat().PAGE_SIZE);
377    * </code>
378    * @usage _intermediate_method_
379    */
380   public int getApproximateOwnedPageCount() {
381 
382     // add a page for the table def (although that might actually be more than
383     // one page)
384     int count = _ownedPages.getPageCount() + 1;
385 
386     for(ColumnImpl col : _columns) {
387       count += col.getOwnedPageCount();
388     }
389 
390     // note, we count owned pages from _physical_ indexes, not logical indexes
391     // (otherwise we could double count pages)
392     for(IndexData indexData : _indexDatas) {
393       count += indexData.getOwnedPageCount();
394     }
395 
396     return count;
397   }
398   
399   protected TempPageHolder getLongValueBuffer() {
400     return _longValueBufferH;
401   }
402 
403   public List<ColumnImpl> getColumns() {
404     return Collections.unmodifiableList(_columns);
405   }
406 
407   public ColumnImpl getColumn(String name) {
408     for(ColumnImpl column : _columns) {
409       if(column.getName().equalsIgnoreCase(name)) {
410         return column;
411       }
412     }
413     throw new IllegalArgumentException(withErrorContext(
414             "Column with name " + name + " does not exist in this table"));
415   }
416   
417   public boolean hasColumn(String name) {
418     for(ColumnImpl column : _columns) {
419       if(column.getName().equalsIgnoreCase(name)) {
420         return true;
421       }
422     }
423     return false;
424   }
425 
426   public PropertyMap getProperties() throws IOException {
427     if(_props == null) {
428       _props = getPropertyMaps().getDefault();
429     }
430     return _props;
431   }
432 
433   /**
434    * @return all PropertyMaps for this table (and columns)
435    * @usage _advanced_method_
436    */
437   public PropertyMaps getPropertyMaps() throws IOException {
438     if(_propertyMaps == null) {
439       _propertyMaps = getDatabase().getPropertiesForObject(
440           _tableDefPageNumber);
441     }
442     return _propertyMaps;
443   }
444   
445   public List<IndexImpl> getIndexes() {
446     return Collections.unmodifiableList(_indexes);
447   }
448 
449   public IndexImpl getIndex(String name) {
450     for(IndexImpl index : _indexes) {
451       if(index.getName().equalsIgnoreCase(name)) {
452         return index;
453       }
454     }
455     throw new IllegalArgumentException(withErrorContext(
456             "Index with name " + name + " does not exist on this table"));
457   }
458 
459   public IndexImpl getPrimaryKeyIndex() {
460     for(IndexImpl index : _indexes) {
461       if(index.isPrimaryKey()) {
462         return index;
463       }
464     }
465     throw new IllegalArgumentException(withErrorContext(
466             "No primary key index found"));
467   }
468   
469   public IndexImpl getForeignKeyIndex(Table otherTable) {
470     for(IndexImpl index : _indexes) {
471       if(index.isForeignKey() && (index.getReference() != null) &&
472          (index.getReference().getOtherTablePageNumber() ==
473           ((TableImpl)otherTable).getTableDefPageNumber())) {
474         return index;
475       }
476     }
477     throw new IllegalArgumentException(withErrorContext(
478         "No foreign key reference to " +
479         otherTable.getName() + " found"));
480   }
481   
482   /**
483    * @return All of the IndexData on this table (unmodifiable List)
484    * @usage _advanced_method_
485    */
486   public List<IndexData> getIndexDatas() {
487     return Collections.unmodifiableList(_indexDatas);
488   }
489 
490   /**
491    * Only called by unit tests
492    * @usage _advanced_method_
493    */
494   public int getLogicalIndexCount() {
495     return _logicalIndexCount;
496   }
497 
498   int getIndexCount() {
499     return _indexCount;
500   }
501 
502   public IndexImpl findIndexForColumns(Collection<String> searchColumns, 
503                                        IndexFeature feature) {
504 
505     IndexImpl partialIndex = null;
506     for(IndexImpl index : _indexes) {
507       
508       Collection<? extends Index.Column> indexColumns = index.getColumns();
509       if(indexColumns.size() < searchColumns.size()) {
510         continue;
511       }
512       boolean exactMatch = (indexColumns.size() == searchColumns.size());
513 
514       Iterator<String> sIter = searchColumns.iterator();
515       Iterator<? extends Index.Column> iIter = indexColumns.iterator();
516       boolean searchMatches = true;
517       while(sIter.hasNext()) {
518         String sColName = sIter.next();
519         String iColName = iIter.next().getName();
520         if((sColName != iColName) &&
521            ((sColName == null) || !sColName.equalsIgnoreCase(iColName))) {
522           searchMatches = false;
523           break;
524         }
525       }
526 
527       if(searchMatches) {
528         
529         if(exactMatch && ((feature != IndexFeature.EXACT_UNIQUE_ONLY) || 
530                           index.isUnique())) {
531           return index;
532         }
533 
534         if(!exactMatch && (feature == IndexFeature.ANY_MATCH) && 
535            ((partialIndex == null) || 
536             (indexColumns.size() < partialIndex.getColumnCount()))) {
537           // this is a better partial index match
538           partialIndex = index;
539         }
540       }
541     }
542 
543     return partialIndex;
544   }
545   
546   List<ColumnImpl> getAutoNumberColumns() {
547     return _autoNumColumns;
548   }
549 
550   public CursorImpl getDefaultCursor() {
551     if(_defaultCursor == null) {
552       _defaultCursor = CursorImpl.createCursor(this);
553     }
554     return _defaultCursor;
555   }
556 
557   public CursorBuilder newCursor() {
558     return new CursorBuilder(this);
559   }
560   
561   public void reset() {
562     getDefaultCursor().reset();
563   }
564 
565   public Row deleteRow(Row row) throws IOException {
566     deleteRow(row.getId());
567     return row;
568   }
569 
570   /**
571    * Delete the row with the given id.  Provided RowId must have previously
572    * been returned from this Table.
573    * @return the given rowId
574    * @throws IllegalStateException if the given row is not valid
575    * @usage _intermediate_method_
576    */
577   public RowId deleteRow(RowId rowId) throws IOException {
578     deleteRow(getDefaultCursor().getRowState(), (RowIdImpl)rowId);
579     return rowId;
580   }
581 
582   /**
583    * Delete the row for the given rowId.
584    * @usage _advanced_method_
585    */
586   public void deleteRow(RowState rowState, RowIdImpl rowId) 
587     throws IOException 
588   {
589     requireValidRowId(rowId);
590     
591     getPageChannel().startWrite();
592     try {
593       
594       // ensure that the relevant row state is up-to-date
595       ByteBuffer rowBuffer = positionAtRowHeader(rowState, rowId);
596 
597       if(rowState.isDeleted()) {
598         // don't care about duplicate deletion
599         return;
600       }
601       requireNonDeletedRow(rowState, rowId);
602     
603       // delete flag always gets set in the "header" row (even if data is on
604       // overflow row)
605       int pageNumber = rowState.getHeaderRowId().getPageNumber();
606       int rowNumber = rowState.getHeaderRowId().getRowNumber();
607 
608       // attempt to fill in index column values
609       Object[] rowValues = null;
610       if(!_indexDatas.isEmpty()) {
611 
612         // move to row data to get index values
613         rowBuffer = positionAtRowData(rowState, rowId);
614 
615         for(ColumnImpl idxCol : _indexColumns) {
616           getRowColumn(getFormat(), rowBuffer, idxCol, rowState, null);
617         }
618 
619         // use any read rowValues to help update the indexes
620         rowValues = rowState.getRowCacheValues();
621 
622         // check foreign keys before proceeding w/ deletion
623         _fkEnforcer.deleteRow(rowValues);    
624 
625         // move back to the header
626         rowBuffer = positionAtRowHeader(rowState, rowId);
627       }
628 
629       // finally, pull the trigger
630       int rowIndex = getRowStartOffset(rowNumber, getFormat());
631       rowBuffer.putShort(rowIndex, (short)(rowBuffer.getShort(rowIndex)
632                                            | DELETED_ROW_MASK | OVERFLOW_ROW_MASK));
633       writeDataPage(rowBuffer, pageNumber);
634 
635       // update the indexes
636       for(IndexData indexData : _indexDatas) {
637         indexData.deleteRow(rowValues, rowId);
638       }
639     
640       // make sure table def gets updated
641       updateTableDefinition(-1);
642 
643     } finally {
644       getPageChannel().finishWrite();
645     }
646   }
647   
648   public Row getNextRow() throws IOException {
649     return getDefaultCursor().getNextRow();
650   }
651   
652   /**
653    * Reads a single column from the given row.
654    * @usage _advanced_method_
655    */
656   public Object getRowValue(RowState rowState, RowIdImpl rowId,
657                             ColumnImpl column)
658     throws IOException
659   {
660     if(this != column.getTable()) {
661       throw new IllegalArgumentException(withErrorContext(
662           "Given column " + column + " is not from this table"));
663     }
664     requireValidRowId(rowId);
665     
666     // position at correct row
667     ByteBuffer rowBuffer = positionAtRowData(rowState, rowId);
668     requireNonDeletedRow(rowState, rowId);
669     
670     return getRowColumn(getFormat(), rowBuffer, column, rowState, null);
671   }
672 
673   /**
674    * Reads some columns from the given row.
675    * @param columnNames Only column names in this collection will be returned
676    * @usage _advanced_method_
677    */
678   public RowImpl getRow(
679       RowState rowState, RowIdImpl rowId, Collection<String> columnNames)
680     throws IOException
681   {
682     requireValidRowId(rowId);
683 
684     // position at correct row
685     ByteBuffer rowBuffer = positionAtRowData(rowState, rowId);
686     requireNonDeletedRow(rowState, rowId);
687 
688     return getRow(getFormat(), rowState, rowBuffer, _columns, columnNames);
689   }
690 
691   /**
692    * Reads the row data from the given row buffer.  Leaves limit unchanged.
693    * Saves parsed row values to the given rowState.
694    */
695   private static RowImpl getRow(
696       JetFormat format,
697       RowState rowState,
698       ByteBuffer rowBuffer,
699       Collection<ColumnImpl> columns,
700       Collection<String> columnNames)
701     throws IOException
702   {
703     RowImpl rtn = new RowImpl(rowState.getHeaderRowId(), columns.size());
704     for(ColumnImpl column : columns) {
705 
706       if((columnNames == null) || (columnNames.contains(column.getName()))) {
707         // Add the value to the row data
708         column.setRowValue(
709             rtn, getRowColumn(format, rowBuffer, column, rowState, null));
710       }
711     }
712     return rtn;
713   }
714   
715   /**
716    * Reads the column data from the given row buffer.  Leaves limit unchanged.
717    * Caches the returned value in the rowState.
718    */
719   private static Object getRowColumn(JetFormat format,
720                                      ByteBuffer rowBuffer,
721                                      ColumnImpl column,
722                                      RowState rowState,
723                                      Map<ColumnImpl,byte[]> rawVarValues)
724     throws IOException
725   {
726     byte[] columnData = null;
727     try {
728 
729       NullMask nullMask = rowState.getNullMask(rowBuffer);
730       boolean isNull = nullMask.isNull(column);
731       if(column.storeInNullMask()) {
732           // Boolean values are stored in the null mask.  see note about
733           // caching below
734         return rowState.setRowCacheValue(column.getColumnIndex(),
735                                          column.readFromNullMask(isNull));
736       } else if(isNull) {
737         // well, that's easy! (no need to update cache w/ null)
738         return null;
739       }
740 
741       Object cachedValue = rowState.getRowCacheValue(column.getColumnIndex());
742       if(cachedValue != null) {
743         // we already have it, use it
744         return cachedValue;
745       }
746       
747       // reset position to row start
748       rowBuffer.reset();
749     
750       // locate the column data bytes
751       int rowStart = rowBuffer.position();
752       int colDataPos = 0;
753       int colDataLen = 0;
754       if(!column.isVariableLength()) {
755 
756         // read fixed length value (non-boolean at this point)
757         int dataStart = rowStart + format.OFFSET_COLUMN_FIXED_DATA_ROW_OFFSET;
758         colDataPos = dataStart + column.getFixedDataOffset();
759         colDataLen = column.getType().getFixedSize(column.getLength());
760       
761       } else {
762         int varDataStart; 
763         int varDataEnd;
764 
765         if(format.SIZE_ROW_VAR_COL_OFFSET == 2) {
766 
767           // read simple var length value
768           int varColumnOffsetPos =
769             (rowBuffer.limit() - nullMask.byteSize() - 4) -
770             (column.getVarLenTableIndex() * 2);
771 
772           varDataStart = rowBuffer.getShort(varColumnOffsetPos);
773           varDataEnd = rowBuffer.getShort(varColumnOffsetPos - 2);
774 
775         } else {
776 
777           // read jump-table based var length values
778           short[] varColumnOffsets = readJumpTableVarColOffsets(
779               rowState, rowBuffer, rowStart, nullMask);
780 
781           varDataStart = varColumnOffsets[column.getVarLenTableIndex()];
782           varDataEnd = varColumnOffsets[column.getVarLenTableIndex() + 1];
783         }
784 
785         colDataPos = rowStart + varDataStart;
786         colDataLen = varDataEnd - varDataStart;
787       }
788 
789       // grab the column data
790       rowBuffer.position(colDataPos);
791       columnData = ByteUtil.getBytes(rowBuffer, colDataLen);
792 
793       if((rawVarValues != null) && column.isVariableLength()) {
794         // caller wants raw value as well
795         rawVarValues.put(column, columnData);
796       }
797 
798       // parse the column data.  we cache the row values in order to be able
799       // to update the index on row deletion.  note, most of the returned
800       // values are immutable, except for binary data (returned as byte[]),
801       // but binary data shouldn't be indexed anyway.
802       return rowState.setRowCacheValue(column.getColumnIndex(), 
803                                        column.read(columnData));
804 
805     } catch(Exception e) {
806 
807       // cache "raw" row value.  see note about caching above
808       rowState.setRowCacheValue(column.getColumnIndex(), 
809                                 ColumnImpl.rawDataWrapper(columnData));
810 
811       return rowState.handleRowError(column, columnData, e);
812     }
813   }
814 
815   private static short[] readJumpTableVarColOffsets(
816       RowState rowState, ByteBuffer rowBuffer, int rowStart,
817       NullMask nullMask) 
818   {
819     short[] varColOffsets = rowState.getVarColOffsets();
820     if(varColOffsets != null) {
821       return varColOffsets;
822     }
823 
824     // calculate offsets using jump-table info
825     int nullMaskSize = nullMask.byteSize();
826     int rowEnd = rowStart + rowBuffer.remaining() - 1;
827     int numVarCols = ByteUtil.getUnsignedByte(rowBuffer, 
828                                               rowEnd - nullMaskSize);
829     varColOffsets = new short[numVarCols + 1];
830 	  
831     int rowLen = rowEnd - rowStart + 1;
832     int numJumps = (rowLen - 1) / MAX_BYTE;
833     int colOffset = rowEnd - nullMaskSize - numJumps - 1;
834 	  
835     // If last jump is a dummy value, ignore it
836     if(((colOffset - rowStart - numVarCols) / MAX_BYTE) < numJumps) {
837       numJumps--;
838     }
839 
840     int jumpsUsed = 0;
841     for(int i = 0; i < numVarCols + 1; i++) {
842 
843       while((jumpsUsed < numJumps) && 
844          (i == ByteUtil.getUnsignedByte(
845               rowBuffer, rowEnd - nullMaskSize-jumpsUsed - 1))) {
846         jumpsUsed++;
847       }
848 		  
849       varColOffsets[i] = (short)
850         (ByteUtil.getUnsignedByte(rowBuffer, colOffset - i)
851          + (jumpsUsed * MAX_BYTE));
852     }
853 	  
854     rowState.setVarColOffsets(varColOffsets);
855     return varColOffsets;
856   }
857 
858   /**
859    * Reads the null mask from the given row buffer.  Leaves limit unchanged.
860    */
861   private NullMask getRowNullMask(ByteBuffer rowBuffer)
862     throws IOException
863   {
864     // reset position to row start
865     rowBuffer.reset();
866 
867     // Number of columns in this row
868     int columnCount = ByteUtil.getUnsignedVarInt(
869         rowBuffer, getFormat().SIZE_ROW_COLUMN_COUNT);
870     
871     // read null mask
872     NullMask nullMask = new NullMask(columnCount);
873     rowBuffer.position(rowBuffer.limit() - nullMask.byteSize());  //Null mask at end
874     nullMask.read(rowBuffer);
875 
876     return nullMask;
877   }
878 
879   /**
880    * Sets a new buffer to the correct row header page using the given rowState
881    * according to the given rowId.  Deleted state is
882    * determined, but overflow row pointers are not followed.
883    * 
884    * @return a ByteBuffer of the relevant page, or null if row was invalid
885    * @usage _advanced_method_
886    */
887   public static ByteBuffer positionAtRowHeader(RowState rowState, 
888                                                RowIdImpl rowId)
889     throws IOException
890   {
891     ByteBuffer rowBuffer = rowState.setHeaderRow(rowId);
892 
893     if(rowState.isAtHeaderRow()) {
894       // this task has already been accomplished
895       return rowBuffer;
896     }
897     
898     if(!rowState.isValid()) {
899       // this was an invalid page/row
900       rowState.setStatus(RowStateStatus.AT_HEADER);
901       return null;
902     }
903 
904     // note, we don't use findRowStart here cause we need the unmasked value
905     short rowStart = rowBuffer.getShort(
906         getRowStartOffset(rowId.getRowNumber(),
907                           rowState.getTable().getFormat()));
908 
909     // check the deleted, overflow flags for the row (the "real" flags are
910     // always set on the header row)
911     RowStatus rowStatus = RowStatus.NORMAL;
912     if(isDeletedRow(rowStart)) {
913       rowStatus = RowStatus.DELETED;
914     } else if(isOverflowRow(rowStart)) {
915       rowStatus = RowStatus.OVERFLOW;
916     }
917 
918     rowState.setRowStatus(rowStatus);
919     rowState.setStatus(RowStateStatus.AT_HEADER);
920     return rowBuffer;
921   }
922   
923   /**
924    * Sets the position and limit in a new buffer using the given rowState
925    * according to the given row number and row end, following overflow row
926    * pointers as necessary.
927    * 
928    * @return a ByteBuffer narrowed to the actual row data, or null if row was
929    *         invalid or deleted
930    * @usage _advanced_method_
931    */
932   public static ByteBuffer positionAtRowData(RowState rowState, 
933                                              RowIdImpl rowId)
934     throws IOException
935   {
936     positionAtRowHeader(rowState, rowId);
937     if(!rowState.isValid() || rowState.isDeleted()) {
938       // row is invalid or deleted
939       rowState.setStatus(RowStateStatus.AT_FINAL);
940       return null;
941     }
942 
943     ByteBuffer rowBuffer = rowState.getFinalPage();
944     int rowNum = rowState.getFinalRowId().getRowNumber();
945     JetFormat format = rowState.getTable().getFormat();
946     
947     if(rowState.isAtFinalRow()) {
948       // we've already found the final row data
949       return PageChannel.narrowBuffer(
950           rowBuffer,
951           findRowStart(rowBuffer, rowNum, format),
952           findRowEnd(rowBuffer, rowNum, format));
953     }
954     
955     while(true) {
956       
957       // note, we don't use findRowStart here cause we need the unmasked value
958       short rowStart = rowBuffer.getShort(getRowStartOffset(rowNum, format));
959       short rowEnd = findRowEnd(rowBuffer, rowNum, format);
960 
961       // note, at this point we know the row is not deleted, so ignore any
962       // subsequent deleted flags (as overflow rows are always marked deleted
963       // anyway)
964       boolean overflowRow = isOverflowRow(rowStart);
965 
966       // now, strip flags from rowStart offset
967       rowStart = (short)(rowStart & OFFSET_MASK);
968 
969       if (overflowRow) {
970 
971         if((rowEnd - rowStart) < 4) {
972           throw new IOException(rowState.getTable().withErrorContext(
973                                     "invalid overflow row info"));
974         }
975       
976         // Overflow page.  the "row" data in the current page points to
977         // another page/row
978         int overflowRowNum = ByteUtil.getUnsignedByte(rowBuffer, rowStart);
979         int overflowPageNum = ByteUtil.get3ByteInt(rowBuffer, rowStart + 1);
980         rowBuffer = rowState.setOverflowRow(
981             new RowIdImpl(overflowPageNum, overflowRowNum));
982         rowNum = overflowRowNum;
983       
984       } else {
985 
986         rowState.setStatus(RowStateStatus.AT_FINAL);
987         return PageChannel.narrowBuffer(rowBuffer, rowStart, rowEnd);
988       }
989     }
990   }
991 
992   public Iterator<Row> iterator() {
993     return getDefaultCursor().iterator();
994   }
995 
996   /**
997    * Writes a new table defined by the given TableCreator to the database.
998    * @usage _advanced_method_
999    */
1000   protected static void writeTableDefinition(TableCreator creator)
1001     throws IOException
1002   {
1003     // first, create the usage map page
1004     createUsageMapDefinitionBuffer(creator);
1005 
1006     // next, determine how big the table def will be (in case it will be more
1007     // than one page)
1008     JetFormat format = creator.getFormat();
1009     int idxDataLen = (creator.getIndexCount() * 
1010                       (format.SIZE_INDEX_DEFINITION + 
1011                        format.SIZE_INDEX_COLUMN_BLOCK)) + 
1012       (creator.getLogicalIndexCount() * format.SIZE_INDEX_INFO_BLOCK);
1013     int colUmapLen = creator.getLongValueColumns().size() * 10;
1014     int totalTableDefSize = format.SIZE_TDEF_HEADER +
1015       (format.SIZE_COLUMN_DEF_BLOCK * creator.getColumns().size()) + 
1016       idxDataLen + colUmapLen + format.SIZE_TDEF_TRAILER;
1017 
1018     // total up the amount of space used by the column and index names (2
1019     // bytes per char + 2 bytes for the length)
1020     for(ColumnBuilder col : creator.getColumns()) {
1021       totalTableDefSize += DBMutator.calculateNameLength(col.getName());
1022     }
1023     
1024     for(IndexBuilder idx : creator.getIndexes()) {
1025       totalTableDefSize += DBMutator.calculateNameLength(idx.getName());
1026     }
1027     
1028 
1029     // now, create the table definition
1030     ByteBuffer buffer = PageChannel.createBuffer(Math.max(totalTableDefSize,
1031                                                           format.PAGE_SIZE));
1032     writeTableDefinitionHeader(creator, buffer, totalTableDefSize);
1033 
1034     if(creator.hasIndexes()) {
1035       // index row counts
1036       IndexData.writeRowCountDefinitions(creator, buffer);
1037     }
1038 
1039     // column definitions
1040     ColumnImpl.writeDefinitions(creator, buffer); 
1041     
1042     if(creator.hasIndexes()) {
1043       // index and index data definitions
1044       IndexData.writeDefinitions(creator, buffer);
1045       IndexImpl.writeDefinitions(creator, buffer);
1046     }
1047 
1048     // column usage map references
1049     ColumnImpl.writeColUsageMapDefinitions(creator, buffer);
1050 
1051     //End of tabledef
1052     buffer.put((byte) 0xff);
1053     buffer.put((byte) 0xff);
1054     buffer.flip();
1055 
1056     // write table buffer to database
1057     writeTableDefinitionBuffer(buffer, creator.getTdefPageNumber(), creator,
1058                                Collections.<Integer>emptyList());
1059   }
1060 
1061   private static void writeTableDefinitionBuffer(
1062       ByteBuffer buffer, int tdefPageNumber, 
1063       TableMutator mutator, List<Integer> reservedPages)
1064     throws IOException
1065   {
1066     buffer.rewind();
1067     int totalTableDefSize = buffer.remaining();
1068     JetFormat format = mutator.getFormat();
1069     PageChannel pageChannel = mutator.getPageChannel();
1070 
1071     // write table buffer to database
1072     if(totalTableDefSize <= format.PAGE_SIZE) {
1073       
1074       // easy case, fits on one page
1075 
1076       // overwrite page free space
1077       buffer.putShort(format.OFFSET_FREE_SPACE,
1078                       (short)(Math.max(
1079                                 format.PAGE_SIZE - totalTableDefSize - 8, 0)));
1080       // Write the tdef page to disk.
1081       buffer.clear();
1082       pageChannel.writePage(buffer, tdefPageNumber);
1083       
1084     } else {
1085 
1086       // need to split across multiple pages
1087 
1088       ByteBuffer partialTdef = pageChannel.createPageBuffer();
1089       buffer.rewind();
1090       int nextTdefPageNumber = PageChannel.INVALID_PAGE_NUMBER;
1091       while(buffer.hasRemaining()) {
1092 
1093         // reset for next write
1094         partialTdef.clear();
1095         
1096         if(nextTdefPageNumber == PageChannel.INVALID_PAGE_NUMBER) {
1097           
1098           // this is the first page.  note, the first page already has the
1099           // page header, so no need to write it here
1100           nextTdefPageNumber = tdefPageNumber;
1101           
1102         } else {
1103 
1104           // write page header
1105           writeTablePageHeader(partialTdef);
1106         }
1107 
1108         // copy the next page of tdef bytes
1109         int curTdefPageNumber = nextTdefPageNumber;
1110         int writeLen = Math.min(partialTdef.remaining(), buffer.remaining());
1111         partialTdef.put(buffer.array(), buffer.position(), writeLen);
1112         ByteUtil.forward(buffer, writeLen);
1113 
1114         if(buffer.hasRemaining()) {
1115           // need a next page
1116           if(reservedPages.isEmpty()) {
1117             nextTdefPageNumber = pageChannel.allocateNewPage();
1118           } else {
1119             nextTdefPageNumber = reservedPages.remove(0);
1120           }
1121           partialTdef.putInt(format.OFFSET_NEXT_TABLE_DEF_PAGE,
1122                              nextTdefPageNumber);
1123         }
1124 
1125         // update page free space
1126         partialTdef.putShort(format.OFFSET_FREE_SPACE,
1127                              (short)(Math.max(
1128                                        partialTdef.remaining() - 8, 0)));
1129 
1130         // write partial page to disk
1131         pageChannel.writePage(partialTdef, curTdefPageNumber);
1132       }
1133         
1134     }
1135     
1136   }
1137 
1138   /**
1139    * Writes a column defined by the given TableUpdater to this table.
1140    * @usage _advanced_method_
1141    */
1142   protected ColumnImpl mutateAddColumn(TableUpdater mutator) throws IOException
1143   {
1144     ColumnBuilder column = mutator.getColumn();
1145     JetFormat format = mutator.getFormat();
1146     boolean isVarCol = column.isVariableLength();
1147     boolean isLongVal = column.getType().isLongValue();
1148 
1149     ////
1150     // calculate how much more space we need in the table def
1151     if(isLongVal) {
1152       mutator.addTdefLen(10);
1153     }
1154 
1155     mutator.addTdefLen(format.SIZE_COLUMN_DEF_BLOCK);
1156 
1157     int nameByteLen = DBMutator.calculateNameLength(column.getName());
1158     mutator.addTdefLen(nameByteLen);
1159 
1160     ////
1161     // load current table definition and add space for new info
1162     ByteBuffer tableBuffer = loadCompleteTableDefinitionBufferForUpdate(
1163         mutator);
1164 
1165     ColumnImpl newCol = null;
1166     int umapPos = -1;
1167     boolean success = false;
1168     try {
1169       
1170       ////
1171       // update various bits of the table def
1172       ByteUtil.forward(tableBuffer, 29);
1173       tableBuffer.putShort((short)(_maxColumnCount + 1));
1174       short varColCount = (short)(_varColumns.size() + (isVarCol ? 1 : 0));
1175       tableBuffer.putShort(varColCount);
1176       tableBuffer.putShort((short)(_columns.size() + 1));
1177 
1178       // move to end of column def blocks
1179       tableBuffer.position(format.SIZE_TDEF_HEADER + 
1180                            (_indexCount * format.SIZE_INDEX_DEFINITION) +
1181                            (_columns.size() * format.SIZE_COLUMN_DEF_BLOCK));
1182 
1183       // figure out the data offsets for the new column
1184       int fixedOffset = 0;
1185       int varOffset = 0;
1186       if(column.isVariableLength()) {
1187         // find the variable offset
1188         for(ColumnImpl col : _varColumns) {
1189           if(col.getVarLenTableIndex() >= varOffset) {
1190             varOffset = col.getVarLenTableIndex() + 1;
1191           }
1192         }
1193       } else {
1194         // find the fixed offset
1195         for(ColumnImpl col : _columns) {
1196           if(!col.isVariableLength() && 
1197              (col.getFixedDataOffset() >= fixedOffset)) {
1198             fixedOffset = col.getFixedDataOffset() + 
1199               col.getType().getFixedSize(col.getLength());
1200           }
1201         }
1202       }
1203 
1204       mutator.setColumnOffsets(fixedOffset, varOffset, varOffset);
1205 
1206       // insert space for the column definition and write it
1207       int colDefPos = tableBuffer.position();
1208       ByteUtil.insertEmptyData(tableBuffer, format.SIZE_COLUMN_DEF_BLOCK);
1209       ColumnImpl.writeDefinition(mutator, column, tableBuffer);
1210 
1211       // skip existing column names and write new name
1212       skipNames(tableBuffer, _columns.size());
1213       ByteUtil.insertEmptyData(tableBuffer, nameByteLen);
1214       writeName(tableBuffer, column.getName(), mutator.getCharset());
1215 
1216       if(isLongVal) {
1217 
1218         // allocate usage maps for the long value col
1219         Map.Entry<Integer,Integer> umapInfo = addUsageMaps(2, null);
1220         TableMutator.ColumnState colState = mutator.getColumnState(column);
1221         colState.setUmapPageNumber(umapInfo.getKey());
1222         byte rowNum = umapInfo.getValue().byteValue();
1223         colState.setUmapOwnedRowNumber(rowNum);
1224         colState.setUmapFreeRowNumber((byte)(rowNum + 1));
1225 
1226         // skip past index defs
1227         ByteUtil.forward(tableBuffer, (_indexCount * 
1228                                        format.SIZE_INDEX_COLUMN_BLOCK));
1229         ByteUtil.forward(tableBuffer,
1230                          (_logicalIndexCount * format.SIZE_INDEX_INFO_BLOCK));
1231         skipNames(tableBuffer, _logicalIndexCount);
1232 
1233         // skip existing usage maps
1234         while(tableBuffer.remaining() >= 2) {
1235           if(tableBuffer.getShort() == IndexData.COLUMN_UNUSED) {
1236             // found end of tdef, we want to insert before this
1237             ByteUtil.forward(tableBuffer, -2);
1238             break;
1239           }
1240         
1241           ByteUtil.forward(tableBuffer, 8);
1242 
1243           // keep reading ...
1244         }
1245 
1246         // write new column usage map info
1247         umapPos = tableBuffer.position();
1248         ByteUtil.insertEmptyData(tableBuffer, 10);
1249         ColumnImpl.writeColUsageMapDefinition(
1250             mutator, column, tableBuffer);
1251       }
1252 
1253       // sanity check the updates
1254       validateTableDefUpdate(mutator, tableBuffer);
1255 
1256       // before writing the new table def, create the column
1257       newCol = ColumnImpl.create(this, tableBuffer, colDefPos,
1258                                  column.getName(), _columns.size());
1259       newCol.setColumnIndex(_columns.size());
1260 
1261       ////
1262       // write updated table def back to the database
1263       writeTableDefinitionBuffer(tableBuffer, _tableDefPageNumber, mutator, 
1264                                  mutator.getNextPages());
1265       success = true;
1266 
1267     } finally {
1268       if(!success) {
1269         // need to discard modified table buffer
1270         _tableDefBufferH.invalidate();
1271       }
1272     }
1273 
1274     ////
1275     // now, update current TableImpl
1276 
1277     _columns.add(newCol);
1278     ++_maxColumnCount;
1279     if(newCol.isVariableLength()) {
1280       _varColumns.add(newCol);
1281       ++_maxVarColumnCount;
1282     }
1283     if(newCol.isAutoNumber()) {
1284       _autoNumColumns.add(newCol);
1285     }
1286 
1287     if(umapPos >= 0) {
1288       // read column usage map
1289       tableBuffer.position(umapPos);
1290       readColumnUsageMaps(tableBuffer);
1291     }
1292 
1293     newCol.postTableLoadInit();
1294 
1295     if(!isSystem()) {
1296       // after fully constructed, allow column validator to be configured (but
1297       // only for user tables)
1298       newCol.setColumnValidator(null);
1299     }
1300 
1301     // save any column properties
1302     Map<String,PropertyMap.Property> colProps = column.getProperties();
1303     if(colProps != null) {
1304       newCol.getProperties().putAll(colProps.values());
1305       getProperties().save();
1306     }
1307 
1308     completeTableMutation(tableBuffer);
1309 
1310     return newCol;
1311   }
1312 
1313   /**
1314    * Writes a index defined by the given TableUpdater to this table.
1315    * @usage _advanced_method_
1316    */
1317   protected IndexData mutateAddIndexData(TableUpdater mutator) throws IOException
1318   {
1319     IndexBuilder index = mutator.getIndex();
1320     JetFormat format = mutator.getFormat();
1321 
1322     ////
1323     // calculate how much more space we need in the table def
1324     mutator.addTdefLen(format.SIZE_INDEX_DEFINITION + 
1325                        format.SIZE_INDEX_COLUMN_BLOCK);
1326 
1327     ////
1328     // load current table definition and add space for new info
1329     ByteBuffer tableBuffer = loadCompleteTableDefinitionBufferForUpdate(
1330         mutator);
1331 
1332     IndexData newIdxData = null;
1333     boolean success = false;
1334     try {
1335       
1336       ////
1337       // update various bits of the table def
1338       ByteUtil.forward(tableBuffer, 39);
1339       tableBuffer.putInt(_indexCount + 1);
1340 
1341       // move to end of index data def blocks
1342       tableBuffer.position(format.SIZE_TDEF_HEADER + 
1343                            (_indexCount * format.SIZE_INDEX_DEFINITION));
1344 
1345       // write index row count definition (empty initially)
1346       ByteUtil.insertEmptyData(tableBuffer, format.SIZE_INDEX_DEFINITION);
1347       IndexData.writeRowCountDefinitions(mutator, tableBuffer, 1);
1348 
1349       // skip columns and column names
1350       ByteUtil.forward(tableBuffer, 
1351                        (_columns.size() * format.SIZE_COLUMN_DEF_BLOCK));
1352       skipNames(tableBuffer, _columns.size());
1353 
1354       // move to end of current index datas
1355       ByteUtil.forward(tableBuffer, (_indexCount * 
1356                                      format.SIZE_INDEX_COLUMN_BLOCK));
1357 
1358       // allocate usage maps and root page
1359       TableMutator.IndexDataState idxDataState = mutator.getIndexDataState(index);
1360       int rootPageNumber = getPageChannel().allocateNewPage();
1361       Map.Entry<Integer,Integer> umapInfo = addUsageMaps(1, rootPageNumber);
1362       idxDataState.setRootPageNumber(rootPageNumber);
1363       idxDataState.setUmapPageNumber(umapInfo.getKey());
1364       idxDataState.setUmapRowNumber(umapInfo.getValue().byteValue());
1365 
1366       // write index data def
1367       int idxDataDefPos = tableBuffer.position();
1368       ByteUtil.insertEmptyData(tableBuffer, format.SIZE_INDEX_COLUMN_BLOCK);
1369       IndexData.writeDefinition(mutator, tableBuffer, idxDataState, null);
1370 
1371       // sanity check the updates
1372       validateTableDefUpdate(mutator, tableBuffer);
1373 
1374       // before writing the new table def, create the index data
1375       tableBuffer.position(0);
1376       newIdxData = IndexData.create(
1377           this, tableBuffer, idxDataState.getIndexDataNumber(), format);
1378       tableBuffer.position(idxDataDefPos);
1379       newIdxData.read(tableBuffer, _columns);
1380 
1381       ////
1382       // write updated table def back to the database
1383       writeTableDefinitionBuffer(tableBuffer, _tableDefPageNumber, mutator, 
1384                                  mutator.getNextPages());
1385       success = true;
1386 
1387     } finally {
1388       if(!success) {
1389         // need to discard modified table buffer
1390         _tableDefBufferH.invalidate();
1391       }
1392     }
1393 
1394     ////
1395     // now, update current TableImpl
1396 
1397     for(IndexData.ColumnDescriptor iCol : newIdxData.getColumns()) {
1398       _indexColumns.add(iCol.getColumn());
1399     }
1400     
1401     ++_indexCount;
1402     _indexDatas.add(newIdxData);
1403 
1404     completeTableMutation(tableBuffer);
1405 
1406     // don't forget to populate the new index
1407     populateIndexData(newIdxData);
1408 
1409     return newIdxData;
1410   }
1411 
1412   private void populateIndexData(IndexData idxData)
1413     throws IOException
1414   {
1415     // grab the columns involved in this index
1416     List<ColumnImpl> idxCols = new ArrayList<ColumnImpl>();
1417     for(IndexData.ColumnDescriptor col : idxData.getColumns()) {
1418       idxCols.add(col.getColumn());
1419     }
1420 
1421     // iterate through all the rows and add them to the index
1422     Object[] rowVals = new Object[_columns.size()];
1423     for(Row row : getDefaultCursor().newIterable().addColumns(idxCols)) {
1424       for(Column col : idxCols) {
1425         col.setRowValue(rowVals, col.getRowValue(row));
1426       }
1427 
1428       IndexData.commitAll(      
1429           idxData.prepareAddRow(rowVals, (RowIdImpl)row.getId(), null));
1430     }
1431 
1432     updateTableDefinition(0);
1433   }
1434 
1435   /**
1436    * Writes a index defined by the given TableUpdater to this table.
1437    * @usage _advanced_method_
1438    */
1439   protected IndexImpl mutateAddIndex(TableUpdater mutator) throws IOException
1440   {
1441     IndexBuilder index = mutator.getIndex();
1442     JetFormat format = mutator.getFormat();
1443 
1444     ////
1445     // calculate how much more space we need in the table def
1446     mutator.addTdefLen(format.SIZE_INDEX_INFO_BLOCK);
1447 
1448     int nameByteLen = DBMutator.calculateNameLength(index.getName());
1449     mutator.addTdefLen(nameByteLen);
1450 
1451     ////
1452     // load current table definition and add space for new info
1453     ByteBuffer tableBuffer = loadCompleteTableDefinitionBufferForUpdate(
1454         mutator);
1455 
1456     IndexImpl newIdx = null;
1457     boolean success = false;
1458     try {
1459       
1460       ////
1461       // update various bits of the table def
1462       ByteUtil.forward(tableBuffer, 35);
1463       tableBuffer.putInt(_logicalIndexCount + 1);
1464 
1465       // move to end of index data def blocks
1466       tableBuffer.position(format.SIZE_TDEF_HEADER + 
1467                            (_indexCount * format.SIZE_INDEX_DEFINITION));
1468 
1469       // skip columns and column names
1470       ByteUtil.forward(tableBuffer, 
1471                        (_columns.size() * format.SIZE_COLUMN_DEF_BLOCK));
1472       skipNames(tableBuffer, _columns.size());
1473 
1474       // move to end of current index datas
1475       ByteUtil.forward(tableBuffer, (_indexCount * 
1476                                      format.SIZE_INDEX_COLUMN_BLOCK));
1477       // move to end of current indexes
1478       ByteUtil.forward(tableBuffer, (_logicalIndexCount * 
1479                                      format.SIZE_INDEX_INFO_BLOCK));
1480 
1481       int idxDefPos = tableBuffer.position();
1482       ByteUtil.insertEmptyData(tableBuffer, format.SIZE_INDEX_INFO_BLOCK);
1483       IndexImpl.writeDefinition(mutator, index, tableBuffer);
1484 
1485       // skip existing index names and write new name
1486       skipNames(tableBuffer, _logicalIndexCount);
1487       ByteUtil.insertEmptyData(tableBuffer, nameByteLen);
1488       writeName(tableBuffer, index.getName(), mutator.getCharset());
1489 
1490       // sanity check the updates
1491       validateTableDefUpdate(mutator, tableBuffer);
1492 
1493       // before writing the new table def, create the index
1494       tableBuffer.position(idxDefPos);
1495       newIdx = new IndexImpl(tableBuffer, _indexDatas, format);
1496       newIdx.setName(index.getName());
1497     
1498       ////
1499       // write updated table def back to the database
1500       writeTableDefinitionBuffer(tableBuffer, _tableDefPageNumber, mutator, 
1501                                  mutator.getNextPages());
1502       success = true;
1503 
1504     } finally {
1505       if(!success) {
1506         // need to discard modified table buffer
1507         _tableDefBufferH.invalidate();
1508       }
1509     }
1510 
1511     ////
1512     // now, update current TableImpl
1513 
1514     ++_logicalIndexCount;
1515     _indexes.add(newIdx);
1516 
1517     completeTableMutation(tableBuffer);
1518 
1519     return newIdx;
1520   }
1521 
1522   private void validateTableDefUpdate(TableUpdater mutator, ByteBuffer tableBuffer)
1523     throws IOException
1524   {
1525     if(!mutator.validateUpdatedTdef(tableBuffer)) {
1526       throw new IllegalStateException(
1527           withErrorContext("Failed updating table definition (unexpected length)"));
1528     }
1529   }
1530 
1531   private void completeTableMutation(ByteBuffer tableBuffer) throws IOException
1532   {
1533     // lastly, may need to clear table def buffer
1534     _tableDefBufferH.possiblyInvalidate(_tableDefPageNumber, tableBuffer);
1535 
1536     // update any foreign key enforcing
1537     _fkEnforcer.reset();
1538 
1539     // update modification count so any active RowStates can keep themselves
1540     // up-to-date
1541     ++_modCount;
1542   }
1543 
1544   /**
1545    * Skips the given number of names in the table buffer.
1546    */
1547   private static void skipNames(ByteBuffer tableBuffer, int count) {
1548     for(int i = 0; i < count; ++i) {
1549       ByteUtil.forward(tableBuffer, tableBuffer.getShort());
1550     }    
1551   }
1552 
1553   private ByteBuffer loadCompleteTableDefinitionBufferForUpdate(
1554       TableUpdater mutator)
1555     throws IOException
1556   {
1557     // load complete table definition
1558     ByteBuffer tableBuffer = _tableDefBufferH.setPage(getPageChannel(),
1559                                                       _tableDefPageNumber);
1560     tableBuffer = loadCompleteTableDefinitionBuffer(
1561         tableBuffer, mutator.getNextPages());
1562 
1563     // make sure the table buffer has enough room for the new info
1564     int addedLen = mutator.getAddedTdefLen();
1565     int origTdefLen = tableBuffer.getInt(8);
1566     mutator.setOrigTdefLen(origTdefLen);
1567     int newTdefLen = origTdefLen + addedLen;
1568     while(newTdefLen > tableBuffer.capacity()) {
1569       tableBuffer = expandTableBuffer(tableBuffer);
1570       tableBuffer.flip();
1571     }
1572 
1573     tableBuffer.limit(origTdefLen);
1574 
1575     // set new tdef length
1576     tableBuffer.position(8);
1577     tableBuffer.putInt(newTdefLen);
1578 
1579     return tableBuffer;
1580   }
1581 
1582   /**
1583    * Adds some usage maps for use with this table.  This method is expected to
1584    * be called with a small-ish number of requested usage maps.
1585    */
1586   private Map.Entry<Integer,Integer> addUsageMaps(
1587       int numMaps, Integer firstUsedPage)
1588     throws IOException
1589   {
1590     JetFormat format = getFormat();
1591     PageChannel pageChannel = getPageChannel();
1592     int umapRowLength = format.OFFSET_USAGE_MAP_START +
1593       format.USAGE_MAP_TABLE_BYTE_LENGTH;
1594     int totalUmapSpaceUsage = getRowSpaceUsage(umapRowLength, format) * numMaps;
1595     int umapPageNumber = PageChannel.INVALID_PAGE_NUMBER;
1596     int firstRowNum = -1;
1597     int freeSpace = 0;
1598 
1599     // search currently known usage map buffers to find one with enough free
1600     // space (the numMaps should always be small enough to put them all on one
1601     // page).  pages will free space will probaby be newer pages (higher
1602     // numbers), so we sort in reverse order.
1603     Set<Integer> knownPages = new TreeSet<Integer>(Collections.reverseOrder());
1604     collectUsageMapPages(knownPages);
1605 
1606     ByteBuffer umapBuf = pageChannel.createPageBuffer();
1607     for(Integer pageNum : knownPages) {
1608       pageChannel.readPage(umapBuf, pageNum);
1609       freeSpace = umapBuf.getShort(format.OFFSET_FREE_SPACE);
1610       if(freeSpace >= totalUmapSpaceUsage) {
1611         // found a page!
1612         umapPageNumber = pageNum;
1613         firstRowNum = getRowsOnDataPage(umapBuf, format);
1614         break;
1615       }
1616     }
1617 
1618     if(umapPageNumber == PageChannel.INVALID_PAGE_NUMBER) {
1619       
1620       // didn't find any existing pages, need to create a new one
1621       umapPageNumber = pageChannel.allocateNewPage();
1622       freeSpace = format.DATA_PAGE_INITIAL_FREE_SPACE;
1623       firstRowNum = 0;
1624       umapBuf = createUsageMapDefPage(pageChannel, freeSpace);
1625     }
1626 
1627     // write the actual usage map defs
1628     int rowStart = findRowEnd(umapBuf, firstRowNum, format) - umapRowLength;
1629     int umapRowNum = firstRowNum;
1630     for(int i = 0; i < numMaps; ++i) {
1631       umapBuf.putShort(getRowStartOffset(umapRowNum, format), (short)rowStart);
1632       umapBuf.put(rowStart, UsageMap.MAP_TYPE_INLINE);
1633 
1634       if(firstUsedPage != null) {
1635         // fill in the first used page of the usage map
1636         umapBuf.putInt(rowStart + 1, firstUsedPage);
1637         umapBuf.put(rowStart + 5, (byte)1);
1638       }
1639 
1640       rowStart -= umapRowLength;      
1641       ++umapRowNum;
1642     }
1643 
1644     // finish the page
1645     freeSpace -= totalUmapSpaceUsage;
1646     umapBuf.putShort(format.OFFSET_FREE_SPACE, (short)freeSpace);
1647     umapBuf.putShort(format.OFFSET_NUM_ROWS_ON_DATA_PAGE, 
1648                      (short)umapRowNum);
1649     pageChannel.writePage(umapBuf, umapPageNumber);
1650 
1651     return new AbstractMap.SimpleImmutableEntry<Integer,Integer>(
1652         umapPageNumber, firstRowNum);
1653   }
1654 
1655   void collectUsageMapPages(Collection<Integer> pages) {
1656     pages.add(_ownedPages.getTablePageNumber());
1657     pages.add(_freeSpacePages.getTablePageNumber());
1658 
1659     for(IndexData idx : _indexDatas) {
1660       idx.collectUsageMapPages(pages);
1661     }
1662 
1663     for(ColumnImpl col : _columns) {
1664       col.collectUsageMapPages(pages);
1665     }
1666   }    
1667   
1668   /**
1669    * @param buffer Buffer to write to
1670    */
1671   private static void writeTableDefinitionHeader(
1672       TableCreator creator, ByteBuffer buffer, int totalTableDefSize)
1673     throws IOException
1674   {
1675     List<ColumnBuilder> columns = creator.getColumns();
1676 
1677     //Start writing the tdef
1678     writeTablePageHeader(buffer);
1679     buffer.putInt(totalTableDefSize);  //Length of table def
1680     buffer.putInt(MAGIC_TABLE_NUMBER); // seemingly constant magic value
1681     buffer.putInt(0);  //Number of rows
1682     buffer.putInt(0); //Last Autonumber
1683     buffer.put((byte) 1); // this makes autonumbering work in access
1684     for (int i = 0; i < 15; i++) {  //Unknown
1685       buffer.put((byte) 0);
1686     }
1687     buffer.put(TYPE_USER); //Table type
1688     buffer.putShort((short) columns.size()); //Max columns a row will have
1689     buffer.putShort(ColumnImpl.countVariableLength(columns));  //Number of variable columns in table
1690     buffer.putShort((short) columns.size()); //Number of columns in table
1691     buffer.putInt(creator.getLogicalIndexCount());  //Number of logical indexes in table
1692     buffer.putInt(creator.getIndexCount());  //Number of indexes in table
1693     buffer.put((byte) 0); //Usage map row number
1694     ByteUtil.put3ByteInt(buffer, creator.getUmapPageNumber());  //Usage map page number
1695     buffer.put((byte) 1); //Free map row number
1696     ByteUtil.put3ByteInt(buffer, creator.getUmapPageNumber());  //Free map page number
1697   }
1698 
1699   /**
1700    * Writes the page header for a table definition page
1701    * @param buffer Buffer to write to
1702    */
1703   private static void writeTablePageHeader(ByteBuffer buffer)
1704   {
1705     buffer.put(PageTypes.TABLE_DEF);  //Page type
1706     buffer.put((byte) 0x01); //Unknown
1707     buffer.put((byte) 0); //Unknown
1708     buffer.put((byte) 0); //Unknown
1709     buffer.putInt(0);  //Next TDEF page pointer
1710   }
1711   
1712   /**
1713    * Writes the given name into the given buffer in the format as expected by
1714    * {@link #readName}.
1715    */
1716   static void writeName(ByteBuffer buffer, String name, Charset charset)
1717   {
1718       ByteBuffer encName = ColumnImpl.encodeUncompressedText(name, charset);
1719       buffer.putShort((short) encName.remaining());
1720       buffer.put(encName);
1721   }
1722   
1723   /**
1724    * Create the usage map definition page buffer.  The "used pages" map is in
1725    * row 0, the "pages with free space" map is in row 1.  Index usage maps are
1726    * in subsequent rows.
1727    */
1728   private static void createUsageMapDefinitionBuffer(TableCreator creator)
1729     throws IOException
1730   {
1731     List<ColumnBuilder> lvalCols = creator.getLongValueColumns();
1732 
1733     // 2 table usage maps plus 1 for each index and 2 for each lval col
1734     int indexUmapEnd = 2 + creator.getIndexCount();
1735     int umapNum = indexUmapEnd + (lvalCols.size() * 2);
1736 
1737     JetFormat format = creator.getFormat();
1738     int umapRowLength = format.OFFSET_USAGE_MAP_START +
1739       format.USAGE_MAP_TABLE_BYTE_LENGTH;
1740     int umapSpaceUsage = getRowSpaceUsage(umapRowLength, format);
1741     PageChannel pageChannel = creator.getPageChannel();
1742     int umapPageNumber = PageChannel.INVALID_PAGE_NUMBER;
1743     ByteBuffer umapBuf = null;
1744     int freeSpace = 0;
1745     int rowStart = 0;
1746     int umapRowNum = 0;
1747     
1748     for(int i = 0; i < umapNum; ++i) {
1749 
1750       if(umapBuf == null) {
1751 
1752         // need new page for usage maps
1753         if(umapPageNumber == PageChannel.INVALID_PAGE_NUMBER) {
1754           // first umap page has already been reserved
1755           umapPageNumber = creator.getUmapPageNumber();
1756         } else {
1757           // need another umap page
1758           umapPageNumber = creator.reservePageNumber();
1759         }
1760 
1761         freeSpace = format.DATA_PAGE_INITIAL_FREE_SPACE;
1762 
1763         umapBuf = createUsageMapDefPage(pageChannel, freeSpace);
1764 
1765         rowStart = findRowEnd(umapBuf, 0, format) - umapRowLength;
1766         umapRowNum = 0;
1767       }
1768 
1769       umapBuf.putShort(getRowStartOffset(umapRowNum, format), (short)rowStart);
1770       
1771       if(i == 0) {
1772 
1773         // table "owned pages" map definition
1774         umapBuf.put(rowStart, UsageMap.MAP_TYPE_REFERENCE);
1775 
1776       } else if(i == 1) {
1777 
1778         // table "free space pages" map definition
1779         umapBuf.put(rowStart, UsageMap.MAP_TYPE_INLINE);
1780 
1781       } else if(i < indexUmapEnd) {
1782 
1783         // index umap
1784         int indexIdx = i - 2;
1785         TableMutator.IndexDataState idxDataState = 
1786           creator.getIndexDataStates().get(indexIdx);
1787         
1788         // allocate root page for the index
1789         int rootPageNumber = pageChannel.allocateNewPage();
1790 
1791         // stash info for later use
1792         idxDataState.setRootPageNumber(rootPageNumber);
1793         idxDataState.setUmapRowNumber((byte)umapRowNum);
1794         idxDataState.setUmapPageNumber(umapPageNumber);
1795 
1796         // index map definition, including initial root page
1797         umapBuf.put(rowStart, UsageMap.MAP_TYPE_INLINE);
1798         umapBuf.putInt(rowStart + 1, rootPageNumber);
1799         umapBuf.put(rowStart + 5, (byte)1);
1800 
1801       } else {
1802 
1803         // long value column umaps
1804         int lvalColIdx = i - indexUmapEnd;
1805         int umapType = lvalColIdx % 2;
1806         lvalColIdx /= 2;
1807 
1808         ColumnBuilder lvalCol = lvalCols.get(lvalColIdx);
1809         TableMutator.ColumnState colState = 
1810           creator.getColumnState(lvalCol);
1811 
1812         umapBuf.put(rowStart, UsageMap.MAP_TYPE_INLINE);
1813 
1814         if((umapType == 1) && 
1815            (umapPageNumber != colState.getUmapPageNumber())) {
1816           // we want to force both usage maps for a column to be on the same
1817           // data page, so just discard the previous one we wrote
1818           --i;
1819           umapType = 0;
1820         }
1821         
1822         if(umapType == 0) {
1823           // lval column "owned pages" usage map
1824           colState.setUmapOwnedRowNumber((byte)umapRowNum);
1825           colState.setUmapPageNumber(umapPageNumber);
1826         } else {
1827           // lval column "free space pages" usage map (always on same page)
1828           colState.setUmapFreeRowNumber((byte)umapRowNum);
1829         }
1830       }
1831 
1832       rowStart -= umapRowLength;
1833       freeSpace -= umapSpaceUsage;
1834       ++umapRowNum;
1835 
1836       if((freeSpace <= umapSpaceUsage) || (i == (umapNum - 1))) {
1837         // finish current page
1838         umapBuf.putShort(format.OFFSET_FREE_SPACE, (short)freeSpace);
1839         umapBuf.putShort(format.OFFSET_NUM_ROWS_ON_DATA_PAGE, 
1840                          (short)umapRowNum);
1841         pageChannel.writePage(umapBuf, umapPageNumber);
1842         umapBuf = null;
1843       }
1844     }
1845   }
1846 
1847   private static ByteBuffer createUsageMapDefPage(
1848       PageChannel pageChannel, int freeSpace)
1849   {
1850     ByteBuffer umapBuf = pageChannel.createPageBuffer();
1851     umapBuf.put(PageTypes.DATA);
1852     umapBuf.put((byte) 0x1);  //Unknown
1853     umapBuf.putShort((short)freeSpace);  //Free space in page
1854     umapBuf.putInt(0); //Table definition
1855     umapBuf.putInt(0); //Unknown
1856     umapBuf.putShort((short)0); //Number of records on this page    
1857     return umapBuf;
1858   }
1859 
1860   /**
1861    * Returns a single ByteBuffer which contains the entire table definition
1862    * (which may span multiple database pages).
1863    */
1864   private ByteBuffer loadCompleteTableDefinitionBuffer(
1865       ByteBuffer tableBuffer, List<Integer> pages)
1866     throws IOException
1867   {
1868     int nextPage = tableBuffer.getInt(getFormat().OFFSET_NEXT_TABLE_DEF_PAGE);
1869     ByteBuffer nextPageBuffer = null;
1870     while (nextPage != 0) {
1871       if(pages != null) {
1872         pages.add(nextPage);
1873       }
1874       if (nextPageBuffer == null) {
1875         nextPageBuffer = getPageChannel().createPageBuffer();
1876       }
1877       getPageChannel().readPage(nextPageBuffer, nextPage);
1878       nextPage = nextPageBuffer.getInt(getFormat().OFFSET_NEXT_TABLE_DEF_PAGE);
1879       tableBuffer = expandTableBuffer(tableBuffer);
1880       tableBuffer.put(nextPageBuffer.array(), 8, getFormat().PAGE_SIZE - 8);
1881       tableBuffer.flip();
1882     }
1883     return tableBuffer;
1884   }
1885     
1886   private ByteBuffer expandTableBuffer(ByteBuffer tableBuffer) {
1887       ByteBuffer newBuffer = PageChannel.createBuffer(
1888           tableBuffer.capacity() + getFormat().PAGE_SIZE - 8);
1889       newBuffer.put(tableBuffer);
1890       return newBuffer;
1891   }
1892     
1893   private void readColumnDefinitions(ByteBuffer tableBuffer, short columnCount)
1894     throws IOException
1895   {
1896     int colOffset = getFormat().OFFSET_INDEX_DEF_BLOCK +
1897         _indexCount * getFormat().SIZE_INDEX_DEFINITION;
1898 
1899     tableBuffer.position(colOffset +
1900                          (columnCount * getFormat().SIZE_COLUMN_HEADER));
1901     List<String> colNames = new ArrayList<String>(columnCount);
1902     for (int i = 0; i < columnCount; i++) {
1903       colNames.add(readName(tableBuffer));
1904     }    
1905     
1906     int dispIndex = 0;
1907     for (int i = 0; i < columnCount; i++) {
1908       ColumnImpl column = ColumnImpl.create(this, tableBuffer,
1909           colOffset + (i * getFormat().SIZE_COLUMN_HEADER), colNames.get(i),
1910           dispIndex++);
1911       _columns.add(column);
1912       if(column.isVariableLength()) {
1913         // also shove it in the variable columns list, which is ordered
1914         // differently from the _columns list
1915         _varColumns.add(column);
1916       }
1917     }
1918 
1919     Collections.sort(_columns);
1920     initAutoNumberColumns();
1921 
1922     // setup the data index for the columns
1923     int colIdx = 0;
1924     for(ColumnImpl col : _columns) {
1925       col.setColumnIndex(colIdx++);
1926     }
1927 
1928     // sort variable length columns based on their index into the variable
1929     // length offset table, because we will write the columns in this order
1930     Collections.sort(_varColumns, VAR_LEN_COLUMN_COMPARATOR);
1931   }
1932 
1933   private void readIndexDefinitions(ByteBuffer tableBuffer) throws IOException
1934   {
1935     // read index column information
1936     for (int i = 0; i < _indexCount; i++) {
1937       IndexData idxData = _indexDatas.get(i);
1938       idxData.read(tableBuffer, _columns);
1939       // keep track of all columns involved in indexes
1940       for(IndexData.ColumnDescriptor iCol : idxData.getColumns()) {
1941         _indexColumns.add(iCol.getColumn());
1942       }
1943     }
1944 
1945     // read logical index info (may be more logical indexes than index datas)
1946     for (int i = 0; i < _logicalIndexCount; i++) {
1947       _indexes.add(new IndexImpl(tableBuffer, _indexDatas, getFormat()));
1948     }
1949 
1950     // read logical index names
1951     for (int i = 0; i < _logicalIndexCount; i++) {
1952       _indexes.get(i).setName(readName(tableBuffer));
1953     }
1954     
1955     Collections.sort(_indexes);
1956   }
1957   
1958   private boolean readColumnUsageMaps(ByteBuffer tableBuffer) 
1959     throws IOException
1960   {
1961     short umapColNum = tableBuffer.getShort();
1962     if(umapColNum == IndexData.COLUMN_UNUSED) {
1963       return false;
1964     }
1965       
1966     int pos = tableBuffer.position();
1967     UsageMap colOwnedPages = null;
1968     UsageMap colFreeSpacePages = null;
1969     try {
1970       colOwnedPages = UsageMap.read(getDatabase(), tableBuffer);
1971       colFreeSpacePages = UsageMap.read(getDatabase(), tableBuffer);
1972     } catch(IllegalStateException e) {
1973       // ignore invalid usage map info
1974       colOwnedPages = null;
1975       colFreeSpacePages = null;
1976       tableBuffer.position(pos + 8);
1977       LOG.warn(withErrorContext("Invalid column " + umapColNum + 
1978                                 " usage map definition: " + e));
1979     }
1980       
1981     for(ColumnImpl col : _columns) {
1982       if(col.getColumnNumber() == umapColNum) {
1983         col.setUsageMaps(colOwnedPages, colFreeSpacePages);
1984         break;
1985       }
1986     }
1987 
1988     return true;
1989   }
1990 
1991   /**
1992    * Writes the given page data to the given page number, clears any other
1993    * relevant buffers.
1994    */
1995   private void writeDataPage(ByteBuffer pageBuffer, int pageNumber)
1996     throws IOException
1997   {
1998     // write the page data
1999     getPageChannel().writePage(pageBuffer, pageNumber);
2000 
2001     // possibly invalidate the add row buffer if a different data buffer is
2002     // being written (e.g. this happens during deleteRow)
2003     _addRowBufferH.possiblyInvalidate(pageNumber, pageBuffer);
2004     
2005     // update modification count so any active RowStates can keep themselves
2006     // up-to-date
2007     ++_modCount;
2008   }
2009 
2010   /**
2011    * Returns a name read from the buffer at the current position. The
2012    * expected name format is the name length followed by the name 
2013    * encoded using the {@link JetFormat#CHARSET}
2014    */
2015   private String readName(ByteBuffer buffer) { 
2016     int nameLength = readNameLength(buffer);
2017     byte[] nameBytes = ByteUtil.getBytes(buffer, nameLength);
2018     return ColumnImpl.decodeUncompressedText(nameBytes, 
2019                                          getDatabase().getCharset());
2020   }
2021   
2022   /**
2023    * Returns a name length read from the buffer at the current position.
2024    */
2025   private int readNameLength(ByteBuffer buffer) { 
2026     return ByteUtil.getUnsignedVarInt(buffer, getFormat().SIZE_NAME_LENGTH);
2027   }
2028   
2029   public Object[] asRow(Map<String,?> rowMap) {
2030     return asRow(rowMap, null, false);
2031   }
2032   
2033   /**
2034    * Converts a map of columnName -> columnValue to an array of row values
2035    * appropriate for a call to {@link #addRow(Object...)}, where the generated
2036    * RowId will be an extra value at the end of the array.
2037    * @see ColumnImpl#RETURN_ROW_ID
2038    * @usage _intermediate_method_
2039    */
2040   public Object[] asRowWithRowId(Map<String,?> rowMap) {
2041     return asRow(rowMap, null, true);
2042   }
2043   
2044   public Object[] asUpdateRow(Map<String,?> rowMap) {
2045     return asRow(rowMap, Column.KEEP_VALUE, false);
2046   }
2047 
2048   /**
2049    * @return the generated RowId added to a row of values created via {@link
2050    *         #asRowWithRowId}
2051    * @usage _intermediate_method_
2052    */
2053   public RowId getRowId(Object[] row) {
2054     return (RowId)row[_columns.size()];
2055   }
2056 
2057   /**
2058    * Converts a map of columnName -> columnValue to an array of row values.
2059    */
2060   private Object[] asRow(Map<String,?> rowMap, Object defaultValue, 
2061                          boolean returnRowId)
2062   {
2063     int len = _columns.size();
2064     if(returnRowId) {
2065       ++len;
2066     }
2067     Object[] row = new Object[len];
2068     if(defaultValue != null) {
2069       Arrays.fill(row, defaultValue);
2070     }
2071     if(returnRowId) {
2072       row[len - 1] = ColumnImpl.RETURN_ROW_ID;
2073     }
2074     if(rowMap == null) {
2075       return row;
2076     }
2077     for(ColumnImpl col : _columns) {
2078       if(rowMap.containsKey(col.getName())) {
2079         col.setRowValue(row, col.getRowValue(rowMap));
2080       }
2081     }
2082     return row;
2083   }
2084   
2085   public Object[] addRow(Object... row) throws IOException {
2086     return addRows(Collections.singletonList(row), false).get(0);
2087   }
2088 
2089   public <M extends Map<String,Object>> M addRowFromMap(M row) 
2090     throws IOException 
2091   {
2092     Object[] rowValues = asRow(row);
2093 
2094     addRow(rowValues);
2095 
2096     returnRowValues(row, rowValues, _autoNumColumns);
2097     return row;
2098   }
2099     
2100   public List<? extends Object[]> addRows(List<? extends Object[]> rows) 
2101     throws IOException 
2102   {
2103     return addRows(rows, true);
2104   }
2105   
2106   public <M extends Map<String,Object>> List<M> addRowsFromMaps(List<M> rows) 
2107     throws IOException 
2108   {
2109     List<Object[]> rowValuesList = new ArrayList<Object[]>(rows.size());
2110     for(Map<String,Object> row : rows) {
2111       rowValuesList.add(asRow(row));
2112     } 
2113 
2114     addRows(rowValuesList);
2115 
2116     if(!_autoNumColumns.isEmpty()) {
2117       for(int i = 0; i < rowValuesList.size(); ++i) {
2118         Map<String,Object> row = rows.get(i);
2119         Object[] rowValues = rowValuesList.get(i);
2120         returnRowValues(row, rowValues, _autoNumColumns);
2121       }
2122     }
2123     return rows;
2124   }
2125 
2126   private static void returnRowValues(Map<String,Object> row, Object[] rowValues,
2127                                       List<ColumnImpl> cols)
2128   {
2129     for(ColumnImpl col : cols) {
2130       col.setRowValue(row, col.getRowValue(rowValues));
2131     }
2132   }
2133 
2134   /**
2135    * Add multiple rows to this table, only writing to disk after all
2136    * rows have been written, and every time a data page is filled.
2137    * @param rows List of Object[] row values
2138    */
2139   private List<? extends Object[]> addRows(List<? extends Object[]> rows,
2140                                            final boolean isBatchWrite)
2141     throws IOException
2142   {
2143     if(rows.isEmpty()) {
2144       return rows;
2145     }
2146 
2147     getPageChannel().startWrite();
2148     try {
2149     
2150       ByteBuffer dataPage = null;
2151       int pageNumber = PageChannel.INVALID_PAGE_NUMBER;
2152       int updateCount = 0;
2153       int autoNumAssignCount = 0;
2154       WriteRowState writeRowState = 
2155         (!_autoNumColumns.isEmpty() ? new WriteRowState() : null);
2156       try {
2157 
2158         List<Object[]> dupeRows = null;
2159         final int numCols = _columns.size();
2160         for (int i = 0; i < rows.size(); i++) {
2161 
2162           // we need to make sure the row is the right length and is an
2163           // Object[] (fill with null if too short).  note, if the row is
2164           // copied the caller will not be able to access any generated
2165           // auto-number value, but if they need that info they should use a
2166           // row array of the right size/type!
2167           Object[] row = rows.get(i);
2168           if((row.length < numCols) || (row.getClass() != Object[].class)) {
2169             row = dupeRow(row, numCols);
2170             // copy the input rows to a modifiable list so we can update the
2171             // elements
2172             if(dupeRows == null) {
2173               dupeRows = new ArrayList<Object[]>(rows);
2174               rows = dupeRows;
2175             }
2176             // we copied the row, so put the copy back into the rows list
2177             dupeRows.set(i, row);
2178           }
2179 
2180           // handle various value massaging activities
2181           for(ColumnImpl column : _columns) {
2182             if(!column.isAutoNumber()) {              
2183               // pass input value through column validator
2184               column.setRowValue(row, column.validate(column.getRowValue(row)));
2185             }
2186           }
2187 
2188           // fill in autonumbers
2189           handleAutoNumbersForAdd(row, writeRowState);
2190           ++autoNumAssignCount;
2191       
2192           // write the row of data to a temporary buffer
2193           ByteBuffer rowData = createRow(
2194               row, _writeRowBufferH.getPageBuffer(getPageChannel()));
2195       
2196           int rowSize = rowData.remaining();
2197           if (rowSize > getFormat().MAX_ROW_SIZE) {
2198             throw new IOException(withErrorContext(
2199                     "Row size " + rowSize + " is too large"));
2200           }
2201 
2202           // get page with space
2203           dataPage = findFreeRowSpace(rowSize, dataPage, pageNumber);
2204           pageNumber = _addRowBufferH.getPageNumber();
2205 
2206           // determine where this row will end up on the page
2207           int rowNum = getRowsOnDataPage(dataPage, getFormat());
2208 
2209           RowIdImpl rowId = new RowIdImpl(pageNumber, rowNum);
2210 
2211           // before we actually write the row data, we verify all the database
2212           // constraints.
2213           if(!_indexDatas.isEmpty()) {
2214 
2215             IndexData.PendingChange idxChange = null;
2216             try {
2217 
2218               // handle foreign keys before adding to table
2219               _fkEnforcer.addRow(row);
2220 
2221               // prepare index updates
2222               for(IndexData indexData : _indexDatas) {
2223                 idxChange = indexData.prepareAddRow(row, rowId, idxChange);
2224               }
2225 
2226               // complete index updates
2227               IndexData.commitAll(idxChange);
2228 
2229             } catch(ConstraintViolationException ce) {
2230               IndexData.rollbackAll(idxChange);
2231               throw ce;
2232             }
2233           }
2234 
2235           // we have satisfied all the constraints, write the row
2236           addDataPageRow(dataPage, rowSize, getFormat(), 0);
2237           dataPage.put(rowData);
2238 
2239           // return rowTd if desired
2240           if((row.length > numCols) && 
2241              (row[numCols] == ColumnImpl.RETURN_ROW_ID)) {
2242             row[numCols] = rowId;
2243           }
2244 
2245           ++updateCount;
2246         }
2247 
2248         writeDataPage(dataPage, pageNumber);
2249     
2250         // Update tdef page
2251         updateTableDefinition(rows.size());
2252 
2253       } catch(Exception rowWriteFailure) {
2254 
2255         boolean isWriteFailure = isWriteFailure(rowWriteFailure);
2256 
2257         if(!isWriteFailure && (autoNumAssignCount > updateCount)) {
2258           // we assigned some autonumbers which won't get written.  attempt to
2259           // recover them so we don't get ugly "holes"
2260           restoreAutoNumbersFromAdd(rows.get(autoNumAssignCount - 1));
2261         }
2262         
2263         if(!isBatchWrite) {
2264           // just re-throw the original exception
2265           if(rowWriteFailure instanceof IOException) {
2266             throw (IOException)rowWriteFailure;
2267           } 
2268           throw (RuntimeException)rowWriteFailure;
2269         }
2270 
2271         // attempt to resolve a partial batch write
2272         if(isWriteFailure) {
2273 
2274           // we don't really know the status of any of the rows, so clear the
2275           // update count
2276           updateCount = 0;
2277 
2278         } else if(updateCount > 0) {
2279           
2280           // attempt to flush the rows already written to disk
2281           try {
2282 
2283             writeDataPage(dataPage, pageNumber);
2284     
2285             // Update tdef page
2286             updateTableDefinition(updateCount);
2287 
2288           } catch(Exception flushFailure) {
2289             // the flush failure is "worse" as it implies possible database
2290             // corruption (failed write vs. a row failure which was not a
2291             // write failure).  we don't know the status of any rows at this
2292             // point (and the original failure is probably irrelevant)
2293             LOG.warn(withErrorContext(
2294                     "Secondary row failure which preceded the write failure"), 
2295                      rowWriteFailure);
2296             updateCount = 0;
2297             rowWriteFailure = flushFailure;
2298           }
2299         }
2300 
2301         throw new BatchUpdateException(
2302             updateCount, withErrorContext("Failed adding rows"),
2303             rowWriteFailure);
2304       }
2305 
2306     } finally {
2307       getPageChannel().finishWrite();
2308     }
2309     
2310     return rows;
2311   }
2312 
2313   private static boolean isWriteFailure(Throwable t) {
2314     while(t != null) {
2315       if((t instanceof IOException) && !(t instanceof JackcessException)) {
2316         return true;
2317       }
2318       t = t.getCause();
2319     } 
2320     // some other sort of exception which is not a write failure
2321     return false;
2322   }
2323   
2324   public Row updateRow(Row row) throws IOException {
2325     return updateRowFromMap(
2326         getDefaultCursor().getRowState(), (RowIdImpl)row.getId(), row);
2327   }
2328 
2329   /**
2330    * Update the row with the given id.  Provided RowId must have previously
2331    * been returned from this Table.
2332    * @return the given row, updated with the current row values
2333    * @throws IllegalStateException if the given row is not valid, or deleted.
2334    * @usage _intermediate_method_
2335    */
2336   public Object[] updateRow(RowId rowId, Object... row) throws IOException {
2337     return updateRow(
2338         getDefaultCursor().getRowState(), (RowIdImpl)rowId, row);
2339   }
2340 
2341   /**
2342    * Update the given column's value for the given row id.  Provided RowId
2343    * must have previously been returned from this Table.
2344    * @throws IllegalStateException if the given row is not valid, or deleted.
2345    * @usage _intermediate_method_
2346    */
2347   public void updateValue(Column column, RowId rowId, Object value) 
2348     throws IOException 
2349   {
2350     Object[] row = new Object[_columns.size()];
2351     Arrays.fill(row, Column.KEEP_VALUE);
2352     column.setRowValue(row, value);
2353 
2354     updateRow(rowId, row);
2355   }
2356 
2357   public <M extends Map<String,Object>> M updateRowFromMap(
2358       RowState rowState, RowIdImpl rowId, M row) 
2359      throws IOException 
2360   {
2361     Object[] rowValues = updateRow(rowState, rowId, asUpdateRow(row));
2362     returnRowValues(row, rowValues, _columns);
2363     return row;
2364   }
2365 
2366   /**
2367    * Update the row for the given rowId.
2368    * @usage _advanced_method_
2369    */
2370   public Object[] updateRow(RowState rowState, RowIdImpl rowId, Object... row) 
2371     throws IOException 
2372   {
2373     requireValidRowId(rowId);
2374     
2375     getPageChannel().startWrite();
2376     try {
2377     
2378       // ensure that the relevant row state is up-to-date
2379       ByteBuffer rowBuffer = positionAtRowData(rowState, rowId);
2380       int oldRowSize = rowBuffer.remaining();
2381 
2382       requireNonDeletedRow(rowState, rowId);
2383 
2384       // we need to make sure the row is the right length & type (fill with
2385       // null if too short).
2386       if((row.length < _columns.size()) || (row.getClass() != Object[].class)) {
2387         row = dupeRow(row, _columns.size());
2388       }
2389 
2390       // hang on to the raw values of var length columns we are "keeping".  this
2391       // will allow us to re-use pre-written var length data, which can save
2392       // space for things like long value columns.
2393       Map<ColumnImpl,byte[]> keepRawVarValues = 
2394         (!_varColumns.isEmpty() ? new HashMap<ColumnImpl,byte[]>() : null);
2395 
2396       // handle various value massaging activities
2397       for(ColumnImpl column : _columns) {
2398 
2399         if(column.isAutoNumber()) {
2400           // handle these separately (below)
2401           continue;
2402         }
2403 
2404         Object rowValue = column.getRowValue(row);
2405         if(rowValue == Column.KEEP_VALUE) {
2406             
2407           // fill in any "keep value" fields (restore old value)
2408           rowValue = getRowColumn(getFormat(), rowBuffer, column, rowState,
2409                                   keepRawVarValues);
2410             
2411         } else {
2412 
2413           // set oldValue to something that could not possibly be a real value
2414           Object oldValue = Column.KEEP_VALUE;
2415           if(_indexColumns.contains(column)) {
2416             // read (old) row value to help update indexes
2417             oldValue = getRowColumn(getFormat(), rowBuffer, column, rowState,
2418                                     null);
2419           } else {
2420             oldValue = rowState.getRowCacheValue(column.getColumnIndex());
2421           }
2422 
2423           // if the old value was passed back in, we don't need to validate
2424           if(oldValue != rowValue) {
2425             // pass input value through column validator
2426             rowValue = column.validate(rowValue);
2427           } 
2428         }
2429 
2430         column.setRowValue(row, rowValue);
2431       }
2432 
2433       // fill in autonumbers
2434       handleAutoNumbersForUpdate(row, rowBuffer, rowState);
2435 
2436       // generate new row bytes
2437       ByteBuffer newRowData = createRow(
2438           row, _writeRowBufferH.getPageBuffer(getPageChannel()), oldRowSize,
2439           keepRawVarValues);
2440 
2441       if (newRowData.limit() > getFormat().MAX_ROW_SIZE) {
2442         throw new IOException(withErrorContext(
2443                 "Row size " + newRowData.limit() + " is too large"));
2444       }
2445 
2446       if(!_indexDatas.isEmpty()) {
2447 
2448         IndexData.PendingChange idxChange = null;
2449         try {
2450 
2451           Object[] oldRowValues = rowState.getRowCacheValues();
2452 
2453           // check foreign keys before actually updating
2454           _fkEnforcer.updateRow(oldRowValues, row);
2455 
2456           // prepare index updates
2457           for(IndexData indexData : _indexDatas) {
2458             idxChange = indexData.prepareUpdateRow(oldRowValues, rowId, row, 
2459                                                    idxChange);
2460           }
2461 
2462           // complete index updates
2463           IndexData.commitAll(idxChange);
2464 
2465         } catch(ConstraintViolationException ce) {
2466           IndexData.rollbackAll(idxChange);
2467           throw ce;
2468         }
2469       }
2470     
2471       // see if we can squeeze the new row data into the existing row
2472       rowBuffer.reset();
2473       int rowSize = newRowData.remaining();
2474 
2475       ByteBuffer dataPage = null;
2476       int pageNumber = PageChannel.INVALID_PAGE_NUMBER;
2477 
2478       if(oldRowSize >= rowSize) {
2479 
2480         // awesome, slap it in!
2481         rowBuffer.put(newRowData);
2482 
2483         // grab the page we just updated
2484         dataPage = rowState.getFinalPage();
2485         pageNumber = rowState.getFinalRowId().getPageNumber();
2486 
2487       } else {
2488 
2489         // bummer, need to find a new page for the data
2490         dataPage = findFreeRowSpace(rowSize, null, 
2491                                     PageChannel.INVALID_PAGE_NUMBER);
2492         pageNumber = _addRowBufferH.getPageNumber();
2493 
2494         RowIdImpl headerRowId = rowState.getHeaderRowId();      
2495         ByteBuffer headerPage = rowState.getHeaderPage();
2496         if(pageNumber == headerRowId.getPageNumber()) {
2497           // new row is on the same page as header row, share page
2498           dataPage = headerPage;
2499         }
2500 
2501         // write out the new row data (set the deleted flag on the new data row
2502         // so that it is ignored during normal table traversal)
2503         int rowNum = addDataPageRow(dataPage, rowSize, getFormat(),
2504                                     DELETED_ROW_MASK);
2505         dataPage.put(newRowData);
2506 
2507         // write the overflow info into the header row and clear out the
2508         // remaining header data
2509         rowBuffer = PageChannel.narrowBuffer(
2510             headerPage,
2511             findRowStart(headerPage, headerRowId.getRowNumber(), getFormat()),
2512             findRowEnd(headerPage, headerRowId.getRowNumber(), getFormat()));
2513         rowBuffer.put((byte)rowNum);
2514         ByteUtil.put3ByteInt(rowBuffer, pageNumber);
2515         ByteUtil.clearRemaining(rowBuffer);
2516 
2517         // set the overflow flag on the header row
2518         int headerRowIndex = getRowStartOffset(headerRowId.getRowNumber(),
2519                                                getFormat());
2520         headerPage.putShort(headerRowIndex,
2521                             (short)(headerPage.getShort(headerRowIndex)
2522                                     | OVERFLOW_ROW_MASK));
2523         if(pageNumber != headerRowId.getPageNumber()) {
2524           writeDataPage(headerPage, headerRowId.getPageNumber());
2525         }
2526       }
2527 
2528       writeDataPage(dataPage, pageNumber);
2529 
2530       updateTableDefinition(0);
2531 
2532     } finally {
2533       getPageChannel().finishWrite();
2534     }
2535 
2536     return row;
2537   }
2538    
2539   private ByteBuffer findFreeRowSpace(int rowSize, ByteBuffer dataPage, 
2540                                       int pageNumber)
2541     throws IOException
2542   {
2543     // assume incoming page is modified
2544     boolean modifiedPage = true;
2545 
2546     if(dataPage == null) {
2547 
2548       // find owned page w/ free space
2549       dataPage = findFreeRowSpace(_ownedPages, _freeSpacePages, 
2550                                   _addRowBufferH);
2551 
2552       if(dataPage == null) {
2553         // No data pages exist (with free space).  Create a new one.
2554         return newDataPage();
2555       }
2556 
2557       // found a page, see if it will work
2558       pageNumber = _addRowBufferH.getPageNumber();
2559       // since we just loaded this page, it is not yet modified
2560       modifiedPage = false;
2561     }
2562 
2563     if(!rowFitsOnDataPage(rowSize, dataPage, getFormat())) {
2564 
2565       // Last data page is full.  Write old one and create a new one.
2566       if(modifiedPage) {
2567         writeDataPage(dataPage, pageNumber);
2568       }
2569       _freeSpacePages.removePageNumber(pageNumber);
2570 
2571       dataPage = newDataPage();
2572     }
2573 
2574     return dataPage;
2575   }
2576 
2577   static ByteBuffer findFreeRowSpace(
2578       UsageMap ownedPages, UsageMap freeSpacePages,
2579       TempPageHolder rowBufferH)
2580     throws IOException
2581   {
2582     // find last data page (Not bothering to check other pages for free
2583     // space.)
2584     UsageMap.PageCursor revPageCursor = ownedPages.cursor();
2585     revPageCursor.afterLast();
2586     while(true) {
2587       int tmpPageNumber = revPageCursor.getPreviousPage();
2588       if(tmpPageNumber < 0) {
2589         break;
2590       }
2591       // only use if actually listed in free space pages
2592       if(!freeSpacePages.containsPageNumber(tmpPageNumber)) {
2593         continue;
2594       }
2595       ByteBuffer dataPage = rowBufferH.setPage(ownedPages.getPageChannel(),
2596                                                tmpPageNumber);
2597       if(dataPage.get() == PageTypes.DATA) {
2598         // found last data page with free space
2599         return dataPage;
2600       }
2601     }
2602 
2603     return null;
2604   }
2605     
2606   /**
2607    * Updates the table definition after rows are modified.
2608    */
2609   private void updateTableDefinition(int rowCountInc) throws IOException
2610   {
2611     // load table definition
2612     ByteBuffer tdefPage = _tableDefBufferH.setPage(getPageChannel(),
2613                                                    _tableDefPageNumber);
2614     
2615     // make sure rowcount and autonumber are up-to-date
2616     _rowCount += rowCountInc;
2617     tdefPage.putInt(getFormat().OFFSET_NUM_ROWS, _rowCount);
2618     tdefPage.putInt(getFormat().OFFSET_NEXT_AUTO_NUMBER, _lastLongAutoNumber);
2619     int ctypeOff = getFormat().OFFSET_NEXT_COMPLEX_AUTO_NUMBER;
2620     if(ctypeOff >= 0) {
2621       tdefPage.putInt(ctypeOff, _lastComplexTypeAutoNumber);
2622     }
2623 
2624     // write any index changes
2625     for (IndexData indexData : _indexDatas) {
2626       // write the unique entry count for the index to the table definition
2627       // page
2628       tdefPage.putInt(indexData.getUniqueEntryCountOffset(),
2629                       indexData.getUniqueEntryCount());
2630       // write the entry page for the index
2631       indexData.update();
2632     }
2633 
2634     // write modified table definition
2635     getPageChannel().writePage(tdefPage, _tableDefPageNumber);
2636   }
2637   
2638   /**
2639    * Create a new data page
2640    * @return Page number of the new page
2641    */
2642   private ByteBuffer newDataPage() throws IOException {
2643     ByteBuffer dataPage = _addRowBufferH.setNewPage(getPageChannel());
2644     dataPage.put(PageTypes.DATA); //Page type
2645     dataPage.put((byte) 1); //Unknown
2646     dataPage.putShort((short)getFormat().DATA_PAGE_INITIAL_FREE_SPACE); //Free space in this page
2647     dataPage.putInt(_tableDefPageNumber); //Page pointer to table definition
2648     dataPage.putInt(0); //Unknown
2649     dataPage.putShort((short)0); //Number of rows on this page
2650     int pageNumber = _addRowBufferH.getPageNumber();
2651     getPageChannel().writePage(dataPage, pageNumber);
2652     _ownedPages.addPageNumber(pageNumber);
2653     _freeSpacePages.addPageNumber(pageNumber);
2654     return dataPage;
2655   }
2656   
2657   protected ByteBuffer createRow(Object[] rowArray, ByteBuffer buffer)
2658     throws IOException
2659   {
2660     return createRow(rowArray, buffer, 0,
2661                      Collections.<ColumnImpl,byte[]>emptyMap());
2662   }
2663 
2664   /**
2665    * Serialize a row of Objects into a byte buffer.
2666    * 
2667    * @param rowArray row data, expected to be correct length for this table
2668    * @param buffer buffer to which to write the row data
2669    * @param minRowSize min size for result row
2670    * @param rawVarValues optional, pre-written values for var length columns
2671    *                     (enables re-use of previously written values).
2672    * @return the given buffer, filled with the row data
2673    */
2674   private ByteBuffer createRow(Object[] rowArray, ByteBuffer buffer,
2675                                int minRowSize, 
2676                                Map<ColumnImpl,byte[]> rawVarValues)
2677     throws IOException
2678   {
2679     buffer.putShort(_maxColumnCount);
2680     NullMask nullMask = new NullMask(_maxColumnCount);
2681     
2682     //Fixed length column data comes first
2683     int fixedDataStart = buffer.position();
2684     int fixedDataEnd = fixedDataStart;
2685     for (ColumnImpl col : _columns) {
2686 
2687       if(col.isVariableLength()) {
2688         continue;
2689       }
2690         
2691       Object rowValue = col.getRowValue(rowArray);
2692 
2693       if (col.storeInNullMask()) {
2694         
2695         if(col.writeToNullMask(rowValue)) {
2696           nullMask.markNotNull(col);
2697         }
2698         rowValue = null;
2699       }
2700           
2701       if(rowValue != null) {
2702         
2703         // we have a value to write
2704         nullMask.markNotNull(col);
2705 
2706         // remainingRowLength is ignored when writing fixed length data
2707         buffer.position(fixedDataStart + col.getFixedDataOffset());
2708         buffer.put(col.write(rowValue, 0));
2709       }
2710 
2711       // always insert space for the entire fixed data column length
2712       // (including null values), access expects the row to always be at least
2713       // big enough to hold all fixed values
2714       buffer.position(fixedDataStart + col.getFixedDataOffset() +
2715                       col.getLength());
2716 
2717       // keep track of the end of fixed data
2718       if(buffer.position() > fixedDataEnd) {
2719         fixedDataEnd = buffer.position();
2720       }                  
2721       
2722     }
2723 
2724     // reposition at end of fixed data
2725     buffer.position(fixedDataEnd);
2726       
2727     // only need this info if this table contains any var length data
2728     if(_maxVarColumnCount > 0) {
2729 
2730       int maxRowSize = getFormat().MAX_ROW_SIZE;
2731 
2732       // figure out how much space remains for var length data.  first,
2733       // account for already written space
2734       maxRowSize -= buffer.position();
2735       // now, account for trailer space
2736       int trailerSize = (nullMask.byteSize() + 4 + (_maxVarColumnCount * 2));
2737       maxRowSize -= trailerSize;
2738 
2739       // for each non-null long value column we need to reserve a small
2740       // amount of space so that we don't end up running out of row space
2741       // later by being too greedy
2742       for (ColumnImpl varCol : _varColumns) {
2743         if((varCol.getType().isLongValue()) &&
2744            (varCol.getRowValue(rowArray) != null)) {
2745           maxRowSize -= getFormat().SIZE_LONG_VALUE_DEF;
2746         }
2747       }
2748       
2749       //Now write out variable length column data
2750       short[] varColumnOffsets = new short[_maxVarColumnCount];
2751       int varColumnOffsetsIndex = 0;
2752       for (ColumnImpl varCol : _varColumns) {
2753         short offset = (short) buffer.position();
2754         Object rowValue = varCol.getRowValue(rowArray);
2755         if (rowValue != null) {
2756           // we have a value
2757           nullMask.markNotNull(varCol);
2758 
2759           byte[] rawValue = null;
2760           ByteBuffer varDataBuf = null;
2761           if(((rawValue = rawVarValues.get(varCol)) != null) && 
2762              (rawValue.length <= maxRowSize)) {
2763             // save time and potentially db space, re-use raw value
2764             varDataBuf = ByteBuffer.wrap(rawValue);
2765           } else {
2766             // write column value
2767             varDataBuf = varCol.write(rowValue, maxRowSize);
2768           }
2769 
2770           maxRowSize -= varDataBuf.remaining();
2771           if(varCol.getType().isLongValue()) {
2772             // we already accounted for some amount of the long value data
2773             // above.  add that space back so we don't double count
2774             maxRowSize += getFormat().SIZE_LONG_VALUE_DEF;
2775           }
2776           try {
2777             buffer.put(varDataBuf);
2778           } catch(BufferOverflowException e) {
2779             // if the data is too big for the buffer, then we have gone over
2780             // the max row size
2781             throw new IOException(withErrorContext(
2782                     "Row size " + buffer.limit() + " is too large"));
2783           } 
2784         }
2785 
2786         // we do a loop here so that we fill in offsets for deleted columns
2787         while(varColumnOffsetsIndex <= varCol.getVarLenTableIndex()) {
2788           varColumnOffsets[varColumnOffsetsIndex++] = offset;
2789         }
2790       }
2791 
2792       // fill in offsets for any remaining deleted columns
2793       while(varColumnOffsetsIndex < varColumnOffsets.length) {
2794         varColumnOffsets[varColumnOffsetsIndex++] = (short) buffer.position();
2795       }
2796 
2797       // record where we stopped writing
2798       int eod = buffer.position();
2799 
2800       // insert padding if necessary
2801       padRowBuffer(buffer, minRowSize, trailerSize);
2802 
2803       buffer.putShort((short) eod); //EOD marker
2804 
2805       //Now write out variable length offsets
2806       //Offsets are stored in reverse order
2807       for (int i = _maxVarColumnCount - 1; i >= 0; i--) {
2808         buffer.putShort(varColumnOffsets[i]);
2809       }
2810       buffer.putShort(_maxVarColumnCount);  //Number of var length columns
2811 
2812     } else {
2813 
2814       // insert padding for row w/ no var cols
2815       padRowBuffer(buffer, minRowSize, nullMask.byteSize());
2816     }
2817 
2818     nullMask.write(buffer);  //Null mask
2819     buffer.flip();
2820     return buffer;
2821   }
2822 
2823   /**
2824    * Fill in all autonumber column values for add.
2825    */
2826   private void handleAutoNumbersForAdd(Object[] row, WriteRowState writeRowState)
2827     throws IOException
2828   {
2829     if(_autoNumColumns.isEmpty()) {
2830       return;
2831     }
2832 
2833     boolean enableInsert = isAllowAutoNumberInsert();
2834     writeRowState.resetAutoNumber();
2835     for(ColumnImpl col : _autoNumColumns) {
2836 
2837       // ignore input row value, use original row value (unless explicitly
2838       // enabled)
2839       Object inRowValue = getInputAutoNumberRowValue(enableInsert, col, row);
2840 
2841       ColumnImpl.AutoNumberGenerator autoNumGen = col.getAutoNumberGenerator();
2842       Object rowValue = ((inRowValue == null) ? 
2843                          autoNumGen.getNext(writeRowState) :
2844                          autoNumGen.handleInsert(writeRowState, inRowValue));
2845 
2846       col.setRowValue(row, rowValue);
2847     }
2848   }
2849   
2850   /**
2851    * Fill in all autonumber column values for update.
2852    */
2853   private void handleAutoNumbersForUpdate(Object[] row, ByteBuffer rowBuffer,
2854                                           RowState rowState)
2855     throws IOException
2856   {
2857     if(_autoNumColumns.isEmpty()) {
2858       return;
2859     }
2860 
2861     boolean enableInsert = isAllowAutoNumberInsert();
2862     rowState.resetAutoNumber();
2863     for(ColumnImpl col : _autoNumColumns) {
2864 
2865       // ignore input row value, use original row value (unless explicitly
2866       // enabled)
2867       Object inRowValue = getInputAutoNumberRowValue(enableInsert, col, row);
2868 
2869       Object rowValue = 
2870         ((inRowValue == null) ?
2871          getRowColumn(getFormat(), rowBuffer, col, rowState, null) :
2872          col.getAutoNumberGenerator().handleInsert(rowState, inRowValue));
2873 
2874       col.setRowValue(row, rowValue);
2875     }
2876   }
2877 
2878   /**
2879    * Optionally get the input autonumber row value for the given column from
2880    * the given row if one was provided.
2881    */
2882   private static Object getInputAutoNumberRowValue(
2883       boolean enableInsert, ColumnImpl col, Object[] row)
2884   {
2885     if(!enableInsert) {
2886       return null;
2887     }
2888 
2889     Object inRowValue = col.getRowValue(row);
2890     if((inRowValue == Column.KEEP_VALUE) || (inRowValue == Column.AUTO_NUMBER)) {
2891       // these "special" values both behave like nothing was given
2892       inRowValue = null;
2893     }
2894     return inRowValue;
2895   }
2896   
2897   /**
2898    * Restores all autonumber column values from a failed add row.
2899    */
2900   private void restoreAutoNumbersFromAdd(Object[] row)
2901     throws IOException
2902   {
2903     if(_autoNumColumns.isEmpty()) {
2904       return;
2905     }
2906 
2907     for(ColumnImpl col : _autoNumColumns) {
2908       // restore the last value from the row
2909       col.getAutoNumberGenerator().restoreLast(col.getRowValue(row));
2910     }
2911   }
2912 
2913   private static void padRowBuffer(ByteBuffer buffer, int minRowSize,
2914                                    int trailerSize)
2915   {
2916     int pos = buffer.position();
2917     if((pos + trailerSize) < minRowSize) {
2918       // pad the row to get to the min byte size
2919       int padSize = minRowSize - (pos + trailerSize);
2920       ByteUtil.clearRange(buffer, pos, pos + padSize);
2921       ByteUtil.forward(buffer, padSize);
2922     }
2923   }
2924 
2925   public int getRowCount() {
2926     return _rowCount;
2927   }
2928 
2929   int getNextLongAutoNumber() {
2930     // note, the saved value is the last one handed out, so pre-increment
2931     return ++_lastLongAutoNumber;
2932   }
2933 
2934   int getLastLongAutoNumber() {
2935     // gets the last used auto number (does not modify)
2936     return _lastLongAutoNumber;
2937   }
2938 
2939   void adjustLongAutoNumber(int inLongAutoNumber) {
2940     if(inLongAutoNumber > _lastLongAutoNumber) {
2941       _lastLongAutoNumber = inLongAutoNumber;
2942     }
2943   }
2944 
2945   void restoreLastLongAutoNumber(int lastLongAutoNumber) {
2946     // restores the last used auto number
2947     _lastLongAutoNumber = lastLongAutoNumber - 1;
2948   }
2949   
2950   int getNextComplexTypeAutoNumber() {
2951     // note, the saved value is the last one handed out, so pre-increment
2952     return ++_lastComplexTypeAutoNumber;
2953   }
2954 
2955   int getLastComplexTypeAutoNumber() {
2956     // gets the last used auto number (does not modify)
2957     return _lastComplexTypeAutoNumber;
2958   }
2959 
2960   void adjustComplexTypeAutoNumber(int inComplexTypeAutoNumber) {
2961     if(inComplexTypeAutoNumber > _lastComplexTypeAutoNumber) {
2962       _lastComplexTypeAutoNumber = inComplexTypeAutoNumber;
2963     }
2964   }
2965 
2966   void restoreLastComplexTypeAutoNumber(int lastComplexTypeAutoNumber) {
2967     // restores the last used auto number
2968     _lastComplexTypeAutoNumber = lastComplexTypeAutoNumber - 1;
2969   }
2970   
2971   @Override
2972   public String toString() {
2973     return CustomToStringStyle.builder(this)
2974       .append("type", (_tableType + (!isSystem() ? " (USER)" : " (SYSTEM)")))
2975       .append("name", _name)
2976       .append("rowCount", _rowCount)
2977       .append("columnCount", _columns.size())
2978       .append("indexCount(data)", _indexCount)
2979       .append("logicalIndexCount", _logicalIndexCount)
2980       .append("columns", _columns)
2981       .append("indexes", _indexes)
2982       .append("ownedPages", _ownedPages)
2983       .toString();
2984   }
2985   
2986   /**
2987    * @return A simple String representation of the entire table in
2988    *         tab-delimited format
2989    * @usage _general_method_
2990    */
2991   public String display() throws IOException {
2992     return display(Long.MAX_VALUE);
2993   }
2994   
2995   /**
2996    * @param limit Maximum number of rows to display
2997    * @return A simple String representation of the entire table in
2998    *         tab-delimited format
2999    * @usage _general_method_
3000    */
3001   public String display(long limit) throws IOException {
3002     reset();
3003     StringWriter rtn = new StringWriter();
3004     new ExportUtil.Builder(getDefaultCursor()).setDelimiter("\t").setHeader(true)
3005       .exportWriter(new BufferedWriter(rtn));
3006     return rtn.toString();
3007   }
3008 
3009   /**
3010    * Updates free space and row info for a new row of the given size in the
3011    * given data page.  Positions the page for writing the row data.
3012    * @return the row number of the new row
3013    * @usage _advanced_method_
3014    */
3015   public static int addDataPageRow(ByteBuffer dataPage,
3016                                    int rowSize,
3017                                    JetFormat format, 
3018                                    int rowFlags)
3019   {
3020     int rowSpaceUsage = getRowSpaceUsage(rowSize, format);
3021     
3022     // Decrease free space record.
3023     short freeSpaceInPage = dataPage.getShort(format.OFFSET_FREE_SPACE);
3024     dataPage.putShort(format.OFFSET_FREE_SPACE, (short) (freeSpaceInPage -
3025                                                          rowSpaceUsage));
3026 
3027     // Increment row count record.
3028     short rowCount = dataPage.getShort(format.OFFSET_NUM_ROWS_ON_DATA_PAGE);
3029     dataPage.putShort(format.OFFSET_NUM_ROWS_ON_DATA_PAGE,
3030                       (short) (rowCount + 1));
3031 
3032     // determine row position
3033     short rowLocation = findRowEnd(dataPage, rowCount, format);
3034     rowLocation -= rowSize;
3035 
3036     // write row position
3037     dataPage.putShort(getRowStartOffset(rowCount, format), 
3038                       (short)(rowLocation | rowFlags));
3039 
3040     // set position for row data
3041     dataPage.position(rowLocation);
3042 
3043     return rowCount;
3044   }
3045 
3046   /**
3047    * Returns the row count for the current page.  If the page is invalid
3048    * ({@code null}) or the page is not a DATA page, 0 is returned.
3049    */
3050   static int getRowsOnDataPage(ByteBuffer rowBuffer, JetFormat format)
3051     throws IOException
3052   {
3053     int rowsOnPage = 0;
3054     if((rowBuffer != null) && (rowBuffer.get(0) == PageTypes.DATA)) {
3055       rowsOnPage = rowBuffer.getShort(format.OFFSET_NUM_ROWS_ON_DATA_PAGE);
3056     }
3057     return rowsOnPage;
3058   }
3059 
3060   /**
3061    * @throws IllegalStateException if the given rowId is invalid
3062    */
3063   private void requireValidRowId(RowIdImpl rowId) {
3064     if(!rowId.isValid()) {
3065       throw new IllegalArgumentException(withErrorContext(
3066               "Given rowId is invalid: " + rowId));
3067     }
3068   }
3069   
3070   /**
3071    * @throws IllegalStateException if the given row is invalid or deleted
3072    */
3073   private void requireNonDeletedRow(RowState rowState, RowIdImpl rowId)
3074   {
3075     if(!rowState.isValid()) {
3076       throw new IllegalArgumentException(withErrorContext(
3077           "Given rowId is invalid for this table: " + rowId));
3078     }
3079     if(rowState.isDeleted()) {
3080       throw new IllegalStateException(withErrorContext(
3081           "Row is deleted: " + rowId));
3082     }
3083   }
3084   
3085   /**
3086    * @usage _advanced_method_
3087    */
3088   public static boolean isDeletedRow(short rowStart) {
3089     return ((rowStart & DELETED_ROW_MASK) != 0);
3090   }
3091   
3092   /**
3093    * @usage _advanced_method_
3094    */
3095   public static boolean isOverflowRow(short rowStart) {
3096     return ((rowStart & OVERFLOW_ROW_MASK) != 0);
3097   }
3098 
3099   /**
3100    * @usage _advanced_method_
3101    */
3102   public static short cleanRowStart(short rowStart) {
3103     return (short)(rowStart & OFFSET_MASK);
3104   }
3105   
3106   /**
3107    * @usage _advanced_method_
3108    */
3109   public static short findRowStart(ByteBuffer buffer, int rowNum,
3110                                    JetFormat format)
3111   {
3112     return cleanRowStart(
3113         buffer.getShort(getRowStartOffset(rowNum, format)));
3114   }
3115 
3116   /**
3117    * @usage _advanced_method_
3118    */
3119   public static int getRowStartOffset(int rowNum, JetFormat format)
3120   {
3121     return format.OFFSET_ROW_START + (format.SIZE_ROW_LOCATION * rowNum);
3122   }
3123   
3124   /**
3125    * @usage _advanced_method_
3126    */
3127   public static short findRowEnd(ByteBuffer buffer, int rowNum,
3128                                  JetFormat format)
3129   {
3130     return (short)((rowNum == 0) ?
3131                    format.PAGE_SIZE :
3132                    cleanRowStart(
3133                        buffer.getShort(getRowEndOffset(rowNum, format))));
3134   }
3135 
3136   /**
3137    * @usage _advanced_method_
3138    */
3139   public static int getRowEndOffset(int rowNum, JetFormat format)
3140   {
3141     return format.OFFSET_ROW_START + (format.SIZE_ROW_LOCATION * (rowNum - 1));
3142   }
3143 
3144   /**
3145    * @usage _advanced_method_
3146    */
3147   public static int getRowSpaceUsage(int rowSize, JetFormat format)
3148   {
3149     return rowSize + format.SIZE_ROW_LOCATION;
3150   }
3151 
3152   private void initAutoNumberColumns() {
3153     for(ColumnImpl c : _columns) {
3154       if(c.isAutoNumber()) {
3155         _autoNumColumns.add(c);
3156       }
3157     }
3158   }
3159 
3160   /**
3161    * Returns {@code true} if a row of the given size will fit on the given
3162    * data page, {@code false} otherwise.
3163    * @usage _advanced_method_
3164    */
3165   public static boolean rowFitsOnDataPage(
3166       int rowLength, ByteBuffer dataPage, JetFormat format)
3167     throws IOException
3168   {
3169     int rowSpaceUsage = getRowSpaceUsage(rowLength, format);
3170     short freeSpaceInPage = dataPage.getShort(format.OFFSET_FREE_SPACE);
3171     int rowsOnPage = getRowsOnDataPage(dataPage, format);
3172     return ((rowSpaceUsage <= freeSpaceInPage) &&
3173             (rowsOnPage < format.MAX_NUM_ROWS_ON_DATA_PAGE));
3174   }
3175 
3176   /**
3177    * Duplicates and returns a row of data, optionally with a longer length
3178    * filled with {@code null}.
3179    */
3180   static Object[] dupeRow(Object[] row, int newRowLength) {
3181     Object[] copy = new Object[newRowLength];
3182     System.arraycopy(row, 0, copy, 0, Math.min(row.length, newRowLength));
3183     return copy;
3184   }
3185 
3186   private String withErrorContext(String msg) {
3187     return withErrorContext(msg, getDatabase(), getName());
3188   }
3189 
3190   private static String withErrorContext(String msg, DatabaseImpl db,
3191                                          String tableName) {
3192     return msg + " (Db=" + db.getName() + ";Table=" + tableName + ")";
3193   }
3194 
3195   /** various statuses for the row data */
3196   private enum RowStatus {
3197     INIT, INVALID_PAGE, INVALID_ROW, VALID, DELETED, NORMAL, OVERFLOW;
3198   }
3199 
3200   /** the phases the RowState moves through as the data is parsed */
3201   private enum RowStateStatus {
3202     INIT, AT_HEADER, AT_FINAL;
3203   }
3204 
3205   /**
3206    * Maintains state for writing a new row of data.
3207    */
3208   protected static class WriteRowState
3209   {
3210     private int _complexAutoNumber = ColumnImpl.INVALID_AUTO_NUMBER;
3211 
3212     public int getComplexAutoNumber() {
3213       return _complexAutoNumber;
3214     }
3215 
3216     public void setComplexAutoNumber(int complexAutoNumber) {
3217       _complexAutoNumber = complexAutoNumber;
3218     }
3219 
3220     public void resetAutoNumber() {
3221       _complexAutoNumber = ColumnImpl.INVALID_AUTO_NUMBER;
3222     }
3223   }
3224 
3225   /**
3226    * Maintains the state of reading/updating a row of data.
3227    * @usage _advanced_class_
3228    */
3229   public final class RowState extends WriteRowState
3230     implements ErrorHandler.Location
3231   {
3232     /** Buffer used for reading the header row data pages */
3233     private final TempPageHolder _headerRowBufferH;
3234     /** the header rowId */
3235     private RowIdImpl _headerRowId = RowIdImpl.FIRST_ROW_ID;
3236     /** the number of rows on the header page */
3237     private int _rowsOnHeaderPage;
3238     /** the rowState status */
3239     private RowStateStatus _status = RowStateStatus.INIT;
3240     /** the row status */
3241     private RowStatus _rowStatus = RowStatus.INIT;
3242     /** buffer used for reading overflow pages */
3243     private final TempPageHolder _overflowRowBufferH =
3244       TempPageHolder.newHolder(TempBufferHolder.Type.SOFT);
3245     /** the row buffer which contains the final data (after following any
3246         overflow pointers) */
3247     private ByteBuffer _finalRowBuffer;
3248     /** the rowId which contains the final data (after following any overflow
3249         pointers) */
3250     private RowIdImpl _finalRowId = null;
3251     /** true if the row values array has data */
3252     private boolean _haveRowValues;
3253     /** values read from the last row */
3254     private Object[] _rowValues;
3255     /** null mask for the last row */
3256     private NullMask _nullMask;
3257     /** last modification count seen on the table we track this so that the
3258         rowState can detect updates to the table and re-read any buffered
3259         data */
3260     private int _lastModCount;
3261     /** optional error handler to use when row errors are encountered */
3262     private ErrorHandler _errorHandler;
3263     /** cached variable column offsets for jump-table based rows */
3264     private short[] _varColOffsets;
3265   
3266     private RowState(TempBufferHolder.Type headerType) {
3267       _headerRowBufferH = TempPageHolder.newHolder(headerType);
3268       _rowValues = new Object[TableImpl.this.getColumnCount()];
3269       _lastModCount = TableImpl.this._modCount;
3270     }
3271 
3272     public TableImpl getTable() {
3273       return TableImpl.this;
3274     }
3275 
3276     public ErrorHandler getErrorHandler() {
3277       return((_errorHandler != null) ? _errorHandler :
3278              getTable().getErrorHandler());
3279     }
3280 
3281     public void setErrorHandler(ErrorHandler newErrorHandler) {
3282       _errorHandler = newErrorHandler;
3283     }
3284     
3285     public void reset() {
3286       resetAutoNumber();
3287       _finalRowId = null;
3288       _finalRowBuffer = null;
3289       _rowsOnHeaderPage = 0;
3290       _status = RowStateStatus.INIT;
3291       _rowStatus = RowStatus.INIT;
3292       _varColOffsets = null;
3293       _nullMask = null;
3294       if(_haveRowValues) {
3295         Arrays.fill(_rowValues, null);
3296         _haveRowValues = false;
3297       }
3298     }
3299 
3300     public boolean isUpToDate() {
3301       return(TableImpl.this._modCount == _lastModCount);
3302     }
3303     
3304     private void checkForModification() {
3305       if(!isUpToDate()) {
3306         reset();
3307         _headerRowBufferH.invalidate();
3308         _overflowRowBufferH.invalidate();
3309         int colCount = TableImpl.this.getColumnCount();
3310         if(colCount != _rowValues.length) {
3311           // columns added or removed from table
3312           _rowValues = new Object[colCount];
3313         }
3314         _lastModCount = TableImpl.this._modCount;
3315       }
3316     }
3317     
3318     private ByteBuffer getFinalPage()
3319       throws IOException
3320     {
3321       if(_finalRowBuffer == null) {
3322         // (re)load current page
3323         _finalRowBuffer = getHeaderPage();
3324       }
3325       return _finalRowBuffer;
3326     }
3327 
3328     public RowIdImpl getFinalRowId() {
3329       if(_finalRowId == null) {
3330         _finalRowId = getHeaderRowId();
3331       }
3332       return _finalRowId;
3333     }
3334 
3335     private void setRowStatus(RowStatus rowStatus) {
3336       _rowStatus = rowStatus;
3337     }
3338 
3339     public boolean isValid() {
3340       return(_rowStatus.ordinal() >= RowStatus.VALID.ordinal());
3341     }
3342     
3343     public boolean isDeleted() {
3344       return(_rowStatus == RowStatus.DELETED);
3345     }
3346     
3347     public boolean isOverflow() {
3348       return(_rowStatus == RowStatus.OVERFLOW);
3349     }
3350 
3351     public boolean isHeaderPageNumberValid() {
3352       return(_rowStatus.ordinal() > RowStatus.INVALID_PAGE.ordinal());
3353     }
3354     
3355     public boolean isHeaderRowNumberValid() {
3356       return(_rowStatus.ordinal() > RowStatus.INVALID_ROW.ordinal());
3357     }
3358     
3359     private void setStatus(RowStateStatus status) {
3360       _status = status;
3361     }
3362     
3363     public boolean isAtHeaderRow() {
3364       return(_status.ordinal() >= RowStateStatus.AT_HEADER.ordinal());
3365     }
3366     
3367     public boolean isAtFinalRow() {
3368       return(_status.ordinal() >= RowStateStatus.AT_FINAL.ordinal());
3369     }
3370 
3371     private Object setRowCacheValue(int idx, Object value) {
3372       _haveRowValues = true;
3373       _rowValues[idx] = value;
3374       return value;
3375     }
3376 
3377     private Object getRowCacheValue(int idx) {
3378       Object value = _rowValues[idx];
3379       // only return immutable values.  mutable values could have been
3380       // modified externally and therefore could return an incorrect value
3381       return(ColumnImpl.isImmutableValue(value) ? value : null);
3382     }
3383     
3384     public Object[] getRowCacheValues() {
3385       return dupeRow(_rowValues, _rowValues.length);
3386     }
3387 
3388     public NullMask getNullMask(ByteBuffer rowBuffer) throws IOException {
3389       if(_nullMask == null) {
3390         _nullMask = getRowNullMask(rowBuffer);
3391       }
3392       return _nullMask;
3393     }
3394 
3395     private short[] getVarColOffsets() {
3396       return _varColOffsets;
3397     }
3398 
3399     private void setVarColOffsets(short[] varColOffsets) {
3400       _varColOffsets = varColOffsets;
3401     }
3402     
3403     public RowIdImpl getHeaderRowId() {
3404       return _headerRowId;
3405     }
3406 
3407     public int getRowsOnHeaderPage() {
3408       return _rowsOnHeaderPage;
3409     }
3410     
3411     private ByteBuffer getHeaderPage()
3412       throws IOException
3413     {
3414       checkForModification();
3415       return _headerRowBufferH.getPage(getPageChannel());
3416     }
3417 
3418     private ByteBuffer setHeaderRow(RowIdImpl rowId)
3419       throws IOException
3420     {
3421       checkForModification();
3422 
3423       // don't do any work if we are already positioned correctly
3424       if(isAtHeaderRow() && (getHeaderRowId().equals(rowId))) {
3425         return(isValid() ? getHeaderPage() : null);
3426       }
3427 
3428       // rejigger everything
3429       reset();
3430       _headerRowId = rowId;
3431       _finalRowId = rowId;
3432 
3433       int pageNumber = rowId.getPageNumber();
3434       int rowNumber = rowId.getRowNumber();
3435       if((pageNumber < 0) || !_ownedPages.containsPageNumber(pageNumber)) {
3436         setRowStatus(RowStatus.INVALID_PAGE);
3437         return null;
3438       }
3439       
3440       _finalRowBuffer = _headerRowBufferH.setPage(getPageChannel(),
3441                                                   pageNumber);
3442       _rowsOnHeaderPage = getRowsOnDataPage(_finalRowBuffer, getFormat());
3443       
3444       if((rowNumber < 0) || (rowNumber >= _rowsOnHeaderPage)) {
3445         setRowStatus(RowStatus.INVALID_ROW);
3446         return null;
3447       }
3448 
3449       setRowStatus(RowStatus.VALID);
3450       return _finalRowBuffer;
3451     }
3452 
3453     private ByteBuffer setOverflowRow(RowIdImpl rowId)
3454       throws IOException
3455     {
3456       // this should never see modifications because it only happens within
3457       // the positionAtRowData method
3458       if(!isUpToDate()) {
3459         throw new IllegalStateException(getTable().withErrorContext(
3460                                             "Table modified while searching?"));
3461       }
3462       if(_rowStatus != RowStatus.OVERFLOW) {
3463         throw new IllegalStateException(getTable().withErrorContext(
3464                                             "Row is not an overflow row?"));
3465       }
3466       _finalRowId = rowId;
3467       _finalRowBuffer = _overflowRowBufferH.setPage(getPageChannel(),
3468                                                     rowId.getPageNumber());
3469       return _finalRowBuffer;
3470     }
3471 
3472     private Object handleRowError(ColumnImpl column, byte[] columnData,
3473                                   Exception error)
3474       throws IOException
3475     {
3476       return getErrorHandler().handleRowError(column, columnData,
3477                                               this, error);
3478     }  
3479 
3480     @Override
3481     public String toString() {
3482       return CustomToStringStyle.valueBuilder(this)
3483         .append("headerRowId", _headerRowId)
3484         .append("finalRowId", _finalRowId)
3485         .toString();
3486     }
3487   }
3488   
3489 }