View Javadoc
1   /*
2   Copyright (c) 2005 Health Market Science, Inc.
3   
4   Licensed under the Apache License, Version 2.0 (the "License");
5   you may not use this file except in compliance with the License.
6   You may obtain a copy of the License at
7   
8       http://www.apache.org/licenses/LICENSE-2.0
9   
10  Unless required by applicable law or agreed to in writing, software
11  distributed under the License is distributed on an "AS IS" BASIS,
12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  See the License for the specific language governing permissions and
14  limitations under the License.
15  */
16  
17  package com.healthmarketscience.jackcess.impl;
18  
19  import java.io.ByteArrayOutputStream;
20  import java.io.IOException;
21  import java.io.InputStream;
22  import java.io.ObjectOutputStream;
23  import java.io.ObjectStreamException;
24  import java.io.Reader;
25  import java.io.Serializable;
26  import java.math.BigDecimal;
27  import java.math.BigInteger;
28  import java.nio.ByteBuffer;
29  import java.nio.ByteOrder;
30  import java.nio.CharBuffer;
31  import java.nio.charset.Charset;
32  import java.sql.Blob;
33  import java.sql.Clob;
34  import java.sql.SQLException;
35  import java.util.Calendar;
36  import java.util.Collection;
37  import java.util.Date;
38  import java.util.List;
39  import java.util.Map;
40  import java.util.UUID;
41  import java.util.regex.Matcher;
42  import java.util.regex.Pattern;
43  
44  import com.healthmarketscience.jackcess.Column;
45  import com.healthmarketscience.jackcess.ColumnBuilder;
46  import com.healthmarketscience.jackcess.DataType;
47  import com.healthmarketscience.jackcess.PropertyMap;
48  import com.healthmarketscience.jackcess.Table;
49  import com.healthmarketscience.jackcess.complex.ComplexColumnInfo;
50  import com.healthmarketscience.jackcess.complex.ComplexValue;
51  import com.healthmarketscience.jackcess.complex.ComplexValueForeignKey;
52  import com.healthmarketscience.jackcess.impl.complex.ComplexValueForeignKeyImpl;
53  import com.healthmarketscience.jackcess.util.ColumnValidator;
54  import com.healthmarketscience.jackcess.util.SimpleColumnValidator;
55  import org.apache.commons.lang.builder.ToStringBuilder;
56  import org.apache.commons.logging.Log;
57  import org.apache.commons.logging.LogFactory;
58  
59  /**
60   * Access database column definition
61   * @author Tim McCune
62   * @usage _intermediate_class_
63   */
64  public class ColumnImpl implements Column, Comparable<ColumnImpl> {
65    
66    protected static final Log LOG = LogFactory.getLog(ColumnImpl.class);
67    
68    /**
69     * Placeholder object for adding rows which indicates that the caller wants
70     * the RowId of the new row.  Must be added as an extra value at the end of
71     * the row values array.
72     * @see TableImpl#asRowWithRowId
73     * @usage _intermediate_field_
74     */
75    public static final Object RETURN_ROW_ID = "<RETURN_ROW_ID>";
76  
77    /**
78     * Access stores numeric dates in days.  Java stores them in milliseconds.
79     */
80    private static final long MILLISECONDS_PER_DAY =
81      (24L * 60L * 60L * 1000L);
82  
83    /**
84     * Access starts counting dates at Dec 30, 1899 (note, this strange date
85     * seems to be caused by MS compatibility with Lotus-1-2-3 and incorrect
86     * leap years).  Java starts counting at Jan 1, 1970.  This is the # of
87     * millis between them for conversion.
88     */
89    static final long MILLIS_BETWEEN_EPOCH_AND_1900 =
90      25569L * MILLISECONDS_PER_DAY;
91    
92    /**
93     * mask for the fixed len bit
94     * @usage _advanced_field_
95     */
96    public static final byte FIXED_LEN_FLAG_MASK = (byte)0x01;
97    
98    /**
99     * mask for the auto number bit
100    * @usage _advanced_field_
101    */
102   public static final byte AUTO_NUMBER_FLAG_MASK = (byte)0x04;
103   
104   /**
105    * mask for the auto number guid bit
106    * @usage _advanced_field_
107    */
108   public static final byte AUTO_NUMBER_GUID_FLAG_MASK = (byte)0x40;
109   
110   /**
111    * mask for the hyperlink bit (on memo types)
112    * @usage _advanced_field_
113    */
114   public static final byte HYPERLINK_FLAG_MASK = (byte)0x80;
115   
116   /**
117    * mask for the "is updatable" field bit
118    * @usage _advanced_field_
119    */
120   public static final byte UPDATABLE_FLAG_MASK = (byte)0x02;
121 
122   // some other flags?
123   // 0x10: replication related field (or hidden?)
124 
125   protected static final byte COMPRESSED_UNICODE_EXT_FLAG_MASK = (byte)0x01;
126   private static final byte CALCULATED_EXT_FLAG_MASK = (byte)0xC0;
127 
128   static final byte NUMERIC_NEGATIVE_BYTE = (byte)0x80;
129 
130   /** the value for the "general" sort order */
131   private static final short GENERAL_SORT_ORDER_VALUE = 1033;
132 
133   /**
134    * the "general" text sort order, legacy version (access 2000-2007)
135    * @usage _intermediate_field_
136    */
137   public static final SortOrder GENERAL_LEGACY_SORT_ORDER =
138     new SortOrder(GENERAL_SORT_ORDER_VALUE, (byte)0);
139 
140   /**
141    * the "general" text sort order, latest version (access 2010+)
142    * @usage _intermediate_field_
143    */
144   public static final SortOrder GENERAL_SORT_ORDER = 
145     new SortOrder(GENERAL_SORT_ORDER_VALUE, (byte)1);
146 
147   /** pattern matching textual guid strings (allows for optional surrounding
148       '{' and '}') */
149   private static final Pattern GUID_PATTERN = Pattern.compile("\\s*[{]?([\\p{XDigit}]{8})-([\\p{XDigit}]{4})-([\\p{XDigit}]{4})-([\\p{XDigit}]{4})-([\\p{XDigit}]{12})[}]?\\s*");
150 
151   /** header used to indicate unicode text compression */
152   private static final byte[] TEXT_COMPRESSION_HEADER = 
153   { (byte)0xFF, (byte)0XFE };
154   private static final char MIN_COMPRESS_CHAR = 1;
155   private static final char MAX_COMPRESS_CHAR = 0xFF;
156 
157   /** auto numbers must be > 0 */
158   static final int INVALID_AUTO_NUMBER = 0;
159 
160 
161   /** owning table */
162   private final TableImpl _table;
163   /** Whether or not the column is of variable length */
164   private final boolean _variableLength;
165   /** Whether or not the column is an autonumber column */
166   private final boolean _autoNumber;
167   /** Whether or not the column is a calculated column */
168   private final boolean _calculated;
169   /** Data type */
170   private final DataType _type;
171   /** Maximum column length */
172   private final short _columnLength;
173   /** 0-based column number */
174   private final short _columnNumber;
175   /** index of the data for this column within a list of row data */
176   private int _columnIndex;
177   /** display index of the data for this column */
178   private final int _displayIndex;
179   /** Column name */
180   private final String _name;
181   /** the offset of the fixed data in the row */
182   private final int _fixedDataOffset;
183   /** the index of the variable length data in the var len offset table */
184   private final int _varLenTableIndex;
185   /** the auto number generator for this column (if autonumber column) */
186   private final AutoNumberGenerator _autoNumberGenerator;
187   /** properties for this column, if any */
188   private PropertyMap _props;  
189   /** Validator for writing new values */
190   private ColumnValidator _validator = SimpleColumnValidator.INSTANCE;
191   
192   /**
193    * @usage _advanced_method_
194    */
195   protected ColumnImpl(TableImpl table, String name, DataType type,
196                        int colNumber, int fixedOffset, int varLenIndex) {
197     _table = table;
198     _name = name;
199     _type = type;
200 
201     if(!_type.isVariableLength()) {
202       _columnLength = (short)type.getFixedSize();
203     } else {
204       _columnLength = (short)type.getMaxSize();
205     }
206     _variableLength = type.isVariableLength();
207     _autoNumber = false;
208     _calculated = false;
209     _autoNumberGenerator = null;
210     _columnNumber = (short)colNumber;
211     _columnIndex = colNumber;
212     _displayIndex = colNumber;
213     _fixedDataOffset = fixedOffset;
214     _varLenTableIndex = varLenIndex;
215   }
216     
217   /**
218    * Read a column definition in from a buffer
219    * @usage _advanced_method_
220    */
221   ColumnImpl(InitArgs args)
222     throws IOException
223   {
224     _table = args.table;
225     _name = args.name;
226     _displayIndex = args.displayIndex;
227     _type = args.type;
228     
229     _columnNumber = args.buffer.getShort(
230         args.offset + getFormat().OFFSET_COLUMN_NUMBER);
231     _columnLength = args.buffer.getShort(
232         args.offset + getFormat().OFFSET_COLUMN_LENGTH);
233     
234     _variableLength = ((args.flags & FIXED_LEN_FLAG_MASK) == 0);
235     _autoNumber = ((args.flags & 
236                     (AUTO_NUMBER_FLAG_MASK | AUTO_NUMBER_GUID_FLAG_MASK)) != 0);
237     _calculated = ((args.extFlags & CALCULATED_EXT_FLAG_MASK) != 0);
238     
239     _autoNumberGenerator = createAutoNumberGenerator();
240     
241     if(_variableLength) {
242       _varLenTableIndex = args.buffer.getShort(
243           args.offset + getFormat().OFFSET_COLUMN_VARIABLE_TABLE_INDEX);
244       _fixedDataOffset = 0;
245     } else {
246       _fixedDataOffset = args.buffer.getShort(
247           args.offset + getFormat().OFFSET_COLUMN_FIXED_DATA_OFFSET);
248       _varLenTableIndex = 0;
249     }
250   }
251   
252   /**
253    * Creates the appropriate ColumnImpl class and reads a column definition in
254    * from a buffer
255    * @param table owning table
256    * @param buffer Buffer containing column definition
257    * @param offset Offset in the buffer at which the column definition starts
258    * @usage _advanced_method_
259    */
260   public static ColumnImpl create(TableImpl table, ByteBuffer buffer,
261                                   int offset, String name, int displayIndex)
262     throws IOException
263   {
264     InitArgs args = new InitArgs(table, buffer, offset, name, displayIndex);
265 
266     boolean calculated = ((args.extFlags & CALCULATED_EXT_FLAG_MASK) != 0);
267     byte colType = args.colType;
268     if(calculated) {
269       // "real" data type is in the "result type" property
270       PropertyMap colProps = table.getPropertyMaps().get(name);
271       Byte resultType = (Byte)colProps.getValue(PropertyMap.RESULT_TYPE_PROP);
272       if(resultType != null) {
273         colType = resultType;
274       }
275     }
276     
277     try {
278       args.type = DataType.fromByte(colType);
279     } catch(IOException e) {
280       LOG.warn(withErrorContext("Unsupported column type " + colType,
281                                 table.getDatabase(), table.getName(), name));
282       boolean variableLength = ((args.flags & FIXED_LEN_FLAG_MASK) == 0);
283       args.type = (variableLength ? DataType.UNSUPPORTED_VARLEN :
284                    DataType.UNSUPPORTED_FIXEDLEN);
285       return new UnsupportedColumnImpl(args);
286     }
287 
288     if(calculated) {
289       return CalculatedColumnUtil.create(args);
290     }
291     
292     switch(args.type) {
293     case TEXT:
294       return new TextColumnImpl(args);
295     case MEMO:
296       return new MemoColumnImpl(args);
297     case COMPLEX_TYPE:
298       return new ComplexColumnImpl(args);
299     default:
300       // fall through
301     }
302 
303     if(args.type.getHasScalePrecision()) {
304       return new NumericColumnImpl(args);
305     }
306     if(args.type.isLongValue()) {
307       return new LongValueColumnImpl(args);
308     }
309     
310     return new ColumnImpl(args);
311   }
312 
313   /**
314    * Sets the usage maps for this column.
315    */
316   void setUsageMaps(UsageMap ownedPages, UsageMap freeSpacePages) {
317     // base does nothing
318   }
319 
320   void collectUsageMapPages(Collection<Integer> pages) {
321     // base does nothing
322   }
323     
324   /**
325    * Secondary column initialization after the table is fully loaded.
326    */
327   void postTableLoadInit() throws IOException {
328     // base does nothing
329   }
330 
331   public TableImpl getTable() {
332     return _table;
333   }
334 
335   public DatabaseImpl getDatabase() {       
336     return getTable().getDatabase();
337   }
338   
339   /**
340    * @usage _advanced_method_
341    */
342   public JetFormat getFormat() {
343     return getDatabase().getFormat();
344   }
345 
346   /**
347    * @usage _advanced_method_
348    */
349   public PageChannel getPageChannel() {
350     return getDatabase().getPageChannel();
351   }
352   
353   public String getName() {
354     return _name;
355   }
356   
357   public boolean isVariableLength() {
358     return _variableLength;
359   }
360   
361   public boolean isAutoNumber() {
362     return _autoNumber;
363   }
364 
365   /**
366    * @usage _advanced_method_
367    */
368   public short getColumnNumber() {
369     return _columnNumber;
370   }
371 
372   public int getColumnIndex() {
373     return _columnIndex;
374   }
375 
376   /**
377    * @usage _advanced_method_
378    */
379   public void setColumnIndex(int newColumnIndex) {
380     _columnIndex = newColumnIndex;
381   }
382   
383   /**
384    * @usage _advanced_method_
385    */
386   public int getDisplayIndex() {
387     return _displayIndex;
388   }
389 
390   public DataType getType() {
391     return _type;
392   }
393   
394   public int getSQLType() throws SQLException {
395     return _type.getSQLType();
396   }
397   
398   public boolean isCompressedUnicode() {
399     return false;
400   }
401 
402   public byte getPrecision() {
403     return (byte)getType().getDefaultPrecision();
404   }
405   
406   public byte getScale() {
407     return (byte)getType().getDefaultScale();
408   }
409 
410   /**
411    * @usage _intermediate_method_
412    */
413   public SortOrder getTextSortOrder() {
414     return null;
415   }
416 
417   /**
418    * @usage _intermediate_method_
419    */
420   public short getTextCodePage() {
421     return 0;
422   }
423 
424   public short getLength() {
425     return _columnLength;
426   }
427 
428   public short getLengthInUnits() {
429     return (short)getType().toUnitSize(getLength());
430   }
431 
432   public boolean isCalculated() {
433     return _calculated;
434   }
435   
436   /**
437    * @usage _advanced_method_
438    */
439   public int getVarLenTableIndex() {
440     return _varLenTableIndex;
441   }
442   
443   /**
444    * @usage _advanced_method_
445    */
446   public int getFixedDataOffset() {
447     return _fixedDataOffset;
448   }
449 
450   protected Charset getCharset() {
451     return getDatabase().getCharset();
452   }
453 
454   protected Calendar getCalendar() {
455     return getDatabase().getCalendar();
456   }
457 
458   public boolean isAppendOnly() {
459     return (getVersionHistoryColumn() != null);
460   }
461   
462   public ColumnImpl getVersionHistoryColumn() {
463     return null;
464   }
465 
466   /**
467    * Returns the number of database pages owned by this column.
468    * @usage _intermediate_method_
469    */
470   public int getOwnedPageCount() {
471     return 0;
472   }
473 
474   /**
475    * @usage _advanced_method_
476    */
477   public void setVersionHistoryColumn(ColumnImpl versionHistoryCol) {
478     throw new UnsupportedOperationException();
479   }
480 
481   public boolean isHyperlink() {
482     return false;
483   }
484   
485   public ComplexColumnInfo<? extends ComplexValue> getComplexInfo() {
486     return null;
487   }
488 
489   public ColumnValidator getColumnValidator() {
490     return _validator;
491   }
492   
493   public void setColumnValidator(ColumnValidator newValidator) {
494     
495     if(isAutoNumber()) {
496       // cannot set autonumber validator (autonumber values are controlled
497       // internally)
498       if(newValidator != null) {
499         throw new IllegalArgumentException(withErrorContext(
500                 "Cannot set ColumnValidator for autonumber columns"));
501       }
502       // just leave default validator instance alone
503       return;
504     }
505     
506     if(newValidator == null) {
507       newValidator = getDatabase().getColumnValidatorFactory()
508         .createValidator(this);
509       if(newValidator == null) {
510         newValidator = SimpleColumnValidator.INSTANCE;
511       }
512     }
513     _validator = newValidator;
514   }
515   
516   byte getOriginalDataType() {
517     return _type.getValue();
518   }
519   
520   private AutoNumberGenerator createAutoNumberGenerator() {
521     if(!_autoNumber || (_type == null)) {
522       return null;
523     }
524 
525     switch(_type) {
526     case LONG:
527       return new LongAutoNumberGenerator();
528     case GUID:
529       return new GuidAutoNumberGenerator();
530     case COMPLEX_TYPE:
531       return new ComplexTypeAutoNumberGenerator();
532     default:
533       LOG.warn(withErrorContext("Unknown auto number column type " + _type));
534       return new UnsupportedAutoNumberGenerator(_type);
535     }
536   }
537 
538   /**
539    * Returns the AutoNumberGenerator for this column if this is an autonumber
540    * column, {@code null} otherwise.
541    * @usage _advanced_method_
542    */
543   public AutoNumberGenerator getAutoNumberGenerator() {
544     return _autoNumberGenerator;
545   }
546 
547   public PropertyMap getProperties() throws IOException {
548     if(_props == null) {
549       _props = getTable().getPropertyMaps().get(getName());
550     }
551     return _props;
552   }
553   
554   public Object setRowValue(Object[] rowArray, Object value) {
555     rowArray[_columnIndex] = value;
556     return value;
557   }
558   
559   public Object setRowValue(Map<String,Object> rowMap, Object value) {
560     rowMap.put(_name, value);
561     return value;
562   }
563   
564   public Object getRowValue(Object[] rowArray) {
565     return rowArray[_columnIndex];
566   }
567   
568   public Object getRowValue(Map<String,?> rowMap) {
569     return rowMap.get(_name);
570   }
571 
572   public boolean storeInNullMask() {
573     return (getType() == DataType.BOOLEAN);
574   }
575   
576   public boolean writeToNullMask(Object value) {
577     return toBooleanValue(value);
578   }
579 
580   public Object readFromNullMask(boolean isNull) {
581     return Boolean.valueOf(!isNull);
582   }
583 
584   /**
585    * Deserialize a raw byte value for this column into an Object
586    * @param data The raw byte value
587    * @return The deserialized Object
588    * @usage _advanced_method_
589    */
590   public Object read(byte[] data) throws IOException {
591     return read(data, PageChannel.DEFAULT_BYTE_ORDER);
592   }
593   
594   /**
595    * Deserialize a raw byte value for this column into an Object
596    * @param data The raw byte value
597    * @param order Byte order in which the raw value is stored
598    * @return The deserialized Object
599    * @usage _advanced_method_
600    */  
601   public Object read(byte[] data, ByteOrder order) throws IOException {
602     ByteBuffer buffer = ByteBuffer.wrap(data).order(order);
603 
604     switch(getType()) {
605     case BOOLEAN:
606       throw new IOException(withErrorContext("Tried to read a boolean from data instead of null mask."));
607     case BYTE:
608       return Byte.valueOf(buffer.get());
609     case INT:
610       return Short.valueOf(buffer.getShort());
611     case LONG:
612       return Integer.valueOf(buffer.getInt());
613     case DOUBLE:
614       return Double.valueOf(buffer.getDouble());
615     case FLOAT:
616       return Float.valueOf(buffer.getFloat());
617     case SHORT_DATE_TIME:
618       return readDateValue(buffer);
619     case BINARY:
620       return data;
621     case TEXT:
622       return decodeTextValue(data);
623     case MONEY:
624       return readCurrencyValue(buffer);
625     case NUMERIC:
626       return readNumericValue(buffer);
627     case GUID:
628       return readGUIDValue(buffer, order);
629     case UNKNOWN_0D:
630     case UNKNOWN_11:
631       // treat like "binary" data
632       return data;
633     case COMPLEX_TYPE:
634       return new ComplexValueForeignKeyImpl(this, buffer.getInt());
635     default:
636       throw new IOException(withErrorContext("Unrecognized data type: " + _type));
637     }
638   }
639 
640   /**
641    * Decodes "Currency" values.
642    * 
643    * @param buffer Column value that points to currency data
644    * @return BigDecimal representing the monetary value
645    * @throws IOException if the value cannot be parsed 
646    */
647   private BigDecimal readCurrencyValue(ByteBuffer buffer)
648     throws IOException
649   {
650     if(buffer.remaining() != 8) {
651       throw new IOException(withErrorContext("Invalid money value"));
652     }
653     
654     return new BigDecimal(BigInteger.valueOf(buffer.getLong(0)), 4);
655   }
656 
657   /**
658    * Writes "Currency" values.
659    */
660   private void writeCurrencyValue(ByteBuffer buffer, Object value)
661     throws IOException
662   {
663     Object inValue = value;
664     try {
665       BigDecimal decVal = toBigDecimal(value);
666       inValue = decVal;
667 
668       // adjust scale (will cause the an ArithmeticException if number has too
669       // many decimal places)
670       decVal = decVal.setScale(4);
671     
672       // now, remove scale and convert to long (this will throw if the value is
673       // too big)
674       buffer.putLong(decVal.movePointRight(4).longValueExact());
675     } catch(ArithmeticException e) {
676       throw (IOException)
677         new IOException(withErrorContext(
678                 "Currency value '" + inValue + "' out of range"))
679         .initCause(e);
680     }
681   }
682 
683   /**
684    * Decodes a NUMERIC field.
685    */
686   private BigDecimal readNumericValue(ByteBuffer buffer)
687   {
688     boolean negate = (buffer.get() != 0);
689 
690     byte[] tmpArr = ByteUtil.getBytes(buffer, 16);
691 
692     if(buffer.order() != ByteOrder.BIG_ENDIAN) {
693       fixNumericByteOrder(tmpArr);
694     }
695 
696     return toBigDecimal(tmpArr, negate, getScale());
697   }
698 
699   static BigDecimal toBigDecimal(byte[] bytes, boolean negate, int scale)
700   {
701     if((bytes[0] & 0x80) != 0) {
702       // the data is effectively unsigned, but the BigInteger handles it as
703       // signed twos complement.  we need to add an extra byte to the input so
704       // that it will be treated as unsigned
705       bytes = ByteUtil.copyOf(bytes, 0, bytes.length + 1, 1);
706     }
707     BigInteger intVal = new BigInteger(bytes);
708     if(negate) {
709       intVal = intVal.negate();
710     }
711     return new BigDecimal(intVal, scale);
712   }
713 
714   /**
715    * Writes a numeric value.
716    */
717   private void writeNumericValue(ByteBuffer buffer, Object value)
718     throws IOException
719   {
720     Object inValue = value;
721     try {
722       BigDecimal decVal = toBigDecimal(value);
723       inValue = decVal;
724 
725       int signum = decVal.signum();
726       if(signum < 0) {
727         decVal = decVal.negate();
728       }
729 
730       // write sign byte
731       buffer.put((signum < 0) ? NUMERIC_NEGATIVE_BYTE : 0);
732 
733       // adjust scale according to this column type (will cause the an
734       // ArithmeticException if number has too many decimal places)
735       decVal = decVal.setScale(getScale());
736 
737       // check precision
738       if(decVal.precision() > getPrecision()) {
739         throw new IOException(withErrorContext(
740             "Numeric value is too big for specified precision "
741             + getPrecision() + ": " + decVal));
742       }
743     
744       // convert to unscaled BigInteger, big-endian bytes
745       byte[] intValBytes = toUnscaledByteArray(
746           decVal, getType().getFixedSize() - 1);
747       if(buffer.order() != ByteOrder.BIG_ENDIAN) {
748         fixNumericByteOrder(intValBytes);
749       }
750       buffer.put(intValBytes);
751     } catch(ArithmeticException e) {
752       throw (IOException)
753         new IOException(withErrorContext(
754                 "Numeric value '" + inValue + "' out of range"))
755         .initCause(e);
756     }
757   }
758 
759   byte[] toUnscaledByteArray(BigDecimal decVal, int maxByteLen)
760     throws IOException
761   {
762     // convert to unscaled BigInteger, big-endian bytes
763     byte[] intValBytes = decVal.unscaledValue().toByteArray();
764     if(intValBytes.length > maxByteLen) {
765       if((intValBytes[0] == 0) && ((intValBytes.length - 1) == maxByteLen)) {
766         // in order to not return a negative two's complement value,
767         // toByteArray() may return an extra leading 0 byte.  we are working
768         // with unsigned values, so we can drop the extra leading 0
769         intValBytes = ByteUtil.copyOf(intValBytes, 1, maxByteLen);
770       } else {
771         throw new IOException(withErrorContext(
772                                   "Too many bytes for valid BigInteger?"));
773       }
774     } else if(intValBytes.length < maxByteLen) {
775       intValBytes = ByteUtil.copyOf(intValBytes, 0, maxByteLen, 
776                                     (maxByteLen - intValBytes.length));
777     }
778     return intValBytes;
779   }
780 
781   /**
782    * Decodes a date value.
783    */
784   private Date readDateValue(ByteBuffer buffer)
785   {
786     // seems access stores dates in the local timezone.  guess you just hope
787     // you read it in the same timezone in which it was written!
788     long dateBits = buffer.getLong();
789     long time = fromDateDouble(Double.longBitsToDouble(dateBits));
790     return new DateExt(time, dateBits);
791   }
792   
793   /**
794    * Returns a java long time value converted from an access date double.
795    * @usage _advanced_method_
796    */
797   public long fromDateDouble(double value)
798   {
799     long localTime = fromLocalDateDouble(value);
800     return localTime - getFromLocalTimeZoneOffset(localTime);
801   }
802 
803   static long fromLocalDateDouble(double value)
804   {
805     long datePart = ((long)value) * MILLISECONDS_PER_DAY;
806 
807     // the fractional part of the double represents the time.  it is always
808     // a positive fraction of the day (even if the double is negative),
809     // _not_ the time distance from zero (as one would expect with "normal"
810     // numbers).  therefore, we need to do a little number logic to convert
811     // the absolute time fraction into a normal distance from zero number.
812     long timePart = Math.round((Math.abs(value) % 1.0) * 
813                                (double)MILLISECONDS_PER_DAY);
814 
815     long time = datePart + timePart;
816     time -= MILLIS_BETWEEN_EPOCH_AND_1900;
817     return time;
818   }
819 
820   /**
821    * Writes a date value.
822    */
823   private void writeDateValue(ByteBuffer buffer, Object value)
824   {
825     if(value == null) {
826       buffer.putDouble(0d);
827     } else if(value instanceof DateExt) {
828       
829       // this is a Date value previously read from readDateValue().  use the
830       // original bits to store the value so we don't lose any precision
831       buffer.putLong(((DateExt)value).getDateBits());
832       
833     } else {
834       
835       buffer.putDouble(toDateDouble(value));
836     }
837   }
838 
839   /**
840    * Returns an access date double converted from a java Date/Calendar/Number
841    * time value.
842    * @usage _advanced_method_
843    */
844   public double toDateDouble(Object value)
845   {
846     // seems access stores dates in the local timezone.  guess you just
847     // hope you read it in the same timezone in which it was written!
848     long time = toDateLong(value);
849     time += getToLocalTimeZoneOffset(time);
850     return toLocalDateDouble(time);
851   }
852 
853   static double toLocalDateDouble(long time)
854   {
855     time += MILLIS_BETWEEN_EPOCH_AND_1900;
856 
857     if(time < 0L) {
858       // reverse the crazy math described in fromLocalDateDouble
859       long timePart = -time % MILLISECONDS_PER_DAY;
860       if(timePart > 0) {
861         time -= (2 * (MILLISECONDS_PER_DAY - timePart));
862       }
863     }
864 
865     return time / (double)MILLISECONDS_PER_DAY;
866   }
867 
868   /**
869    * @return an appropriate Date long value for the given object
870    */
871   private static long toDateLong(Object value) 
872   {
873     return ((value instanceof Date) ?
874             ((Date)value).getTime() :
875             ((value instanceof Calendar) ?
876              ((Calendar)value).getTimeInMillis() :
877              ((Number)value).longValue()));
878   }
879 
880   /**
881    * Gets the timezone offset from UTC to local time for the given time
882    * (including DST).
883    */
884   private long getToLocalTimeZoneOffset(long time)
885   {
886     Calendar c = getCalendar();
887     c.setTimeInMillis(time);
888     return ((long)c.get(Calendar.ZONE_OFFSET) + c.get(Calendar.DST_OFFSET));
889   }  
890   
891   /**
892    * Gets the timezone offset from local time to UTC for the given time
893    * (including DST).
894    */
895   private long getFromLocalTimeZoneOffset(long time)
896   {
897     // getting from local time back to UTC is a little wonky (and not
898     // guaranteed to get you back to where you started)
899     Calendar c = getCalendar();
900     c.setTimeInMillis(time);
901     // apply the zone offset first to get us closer to the original time
902     c.setTimeInMillis(time - c.get(Calendar.ZONE_OFFSET));
903     return ((long)c.get(Calendar.ZONE_OFFSET) + c.get(Calendar.DST_OFFSET));
904   }  
905   
906   /**
907    * Decodes a GUID value.
908    */
909   private static String readGUIDValue(ByteBuffer buffer, ByteOrder order)
910   {
911     if(order != ByteOrder.BIG_ENDIAN) {
912       byte[] tmpArr = ByteUtil.getBytes(buffer, 16);
913 
914         // the first 3 guid components are integer components which need to
915         // respect endianness, so swap 4-byte int, 2-byte int, 2-byte int
916       ByteUtil.swap4Bytes(tmpArr, 0);
917       ByteUtil.swap2Bytes(tmpArr, 4);
918       ByteUtil.swap2Bytes(tmpArr, 6);
919       buffer = ByteBuffer.wrap(tmpArr);
920     }
921 
922     StringBuilder sb = new StringBuilder(22);
923     sb.append("{");
924     sb.append(ByteUtil.toHexString(buffer, 0, 4,
925                                    false));
926     sb.append("-");
927     sb.append(ByteUtil.toHexString(buffer, 4, 2,
928                                    false));
929     sb.append("-");
930     sb.append(ByteUtil.toHexString(buffer, 6, 2,
931                                    false));
932     sb.append("-");
933     sb.append(ByteUtil.toHexString(buffer, 8, 2,
934                                    false));
935     sb.append("-");
936     sb.append(ByteUtil.toHexString(buffer, 10, 6,
937                                    false));
938     sb.append("}");
939     return (sb.toString());
940   }
941 
942   /**
943    * Writes a GUID value.
944    */
945   private void writeGUIDValue(ByteBuffer buffer, Object value)
946     throws IOException
947   {
948     Matcher m = GUID_PATTERN.matcher(toCharSequence(value));
949     if(!m.matches()) {
950       throw new IOException(withErrorContext("Invalid GUID: " + value));
951     }
952 
953     ByteBuffer origBuffer = null;
954     byte[] tmpBuf = null;
955     if(buffer.order() != ByteOrder.BIG_ENDIAN) {
956       // write to a temp buf so we can do some swapping below
957       origBuffer = buffer;
958       tmpBuf = new byte[16];
959       buffer = ByteBuffer.wrap(tmpBuf);
960     }
961 
962     ByteUtil.writeHexString(buffer, m.group(1));
963     ByteUtil.writeHexString(buffer, m.group(2));
964     ByteUtil.writeHexString(buffer, m.group(3));
965     ByteUtil.writeHexString(buffer, m.group(4));
966     ByteUtil.writeHexString(buffer, m.group(5));
967       
968     if(tmpBuf != null) {
969       // the first 3 guid components are integer components which need to
970       // respect endianness, so swap 4-byte int, 2-byte int, 2-byte int
971       ByteUtil.swap4Bytes(tmpBuf, 0);
972       ByteUtil.swap2Bytes(tmpBuf, 4);
973       ByteUtil.swap2Bytes(tmpBuf, 6);
974       origBuffer.put(tmpBuf);
975     }
976   }
977 
978   /**
979    * Returns {@code true} if the given value is a "guid" value.
980    */
981   static boolean isGUIDValue(Object value) throws IOException {
982     return GUID_PATTERN.matcher(toCharSequence(value)).matches();
983   }
984 
985   /**
986    * Passes the given obj through the currently configured validator for this
987    * column and returns the result.
988    */
989   public Object validate(Object obj) throws IOException {
990     return _validator.validate(this, obj);
991   }
992   
993   /**
994    * Serialize an Object into a raw byte value for this column in little
995    * endian order
996    * @param obj Object to serialize
997    * @return A buffer containing the bytes
998    * @usage _advanced_method_
999    */
1000   public ByteBuffer write(Object obj, int remainingRowLength)
1001     throws IOException
1002   {
1003     return write(obj, remainingRowLength, PageChannel.DEFAULT_BYTE_ORDER);
1004   }
1005   
1006   /**
1007    * Serialize an Object into a raw byte value for this column
1008    * @param obj Object to serialize
1009    * @param order Order in which to serialize
1010    * @return A buffer containing the bytes
1011    * @usage _advanced_method_
1012    */
1013   public ByteBuffer write(Object obj, int remainingRowLength, ByteOrder order)
1014     throws IOException
1015   {
1016     if(isRawData(obj)) {
1017       // just slap it right in (not for the faint of heart!)
1018       return ByteBuffer.wrap(((RawData)obj).getBytes());
1019     }
1020 
1021     return writeRealData(obj, remainingRowLength, order);
1022   }
1023 
1024   protected ByteBuffer writeRealData(Object obj, int remainingRowLength, 
1025                                      ByteOrder order)
1026     throws IOException
1027   {
1028     if(!isVariableLength() || !getType().isVariableLength()) {
1029       return writeFixedLengthField(obj, order);
1030     }
1031       
1032     // this is an "inline" var length field
1033     switch(getType()) {
1034     case NUMERIC:
1035       // don't ask me why numerics are "var length" columns...
1036       ByteBuffer buffer = PageChannel.createBuffer(
1037           getType().getFixedSize(), order);
1038       writeNumericValue(buffer, obj);
1039       buffer.flip();
1040       return buffer;
1041 
1042     case TEXT:
1043       return encodeTextValue(
1044           obj, 0, getLengthInUnits(), false).order(order);
1045         
1046     case BINARY:
1047     case UNKNOWN_0D:
1048     case UNSUPPORTED_VARLEN:
1049       // should already be "encoded"
1050       break;
1051     default:
1052       throw new RuntimeException(withErrorContext(
1053               "unexpected inline var length type: " + getType()));
1054     }
1055 
1056     ByteBuffer buffer = ByteBuffer.wrap(toByteArray(obj)).order(order);
1057     return buffer;
1058   }
1059 
1060   /**
1061    * Serialize an Object into a raw byte value for this column
1062    * @param obj Object to serialize
1063    * @param order Order in which to serialize
1064    * @return A buffer containing the bytes
1065    * @usage _advanced_method_
1066    */
1067   protected ByteBuffer writeFixedLengthField(Object obj, ByteOrder order)
1068     throws IOException
1069   {
1070     int size = getType().getFixedSize(_columnLength);
1071 
1072     ByteBuffer buffer = writeFixedLengthField(
1073         obj, PageChannel.createBuffer(size, order));
1074     buffer.flip();
1075     return buffer;
1076   }
1077 
1078   protected ByteBuffer writeFixedLengthField(Object obj, ByteBuffer buffer)
1079     throws IOException
1080   {
1081     // since booleans are not written by this method, it's safe to convert any
1082     // incoming boolean into an integer.
1083     obj = booleanToInteger(obj);
1084 
1085     switch(getType()) {
1086     case BOOLEAN:
1087       //Do nothing
1088       break;
1089     case  BYTE:
1090       buffer.put(toNumber(obj).byteValue());
1091       break;
1092     case INT:
1093       buffer.putShort(toNumber(obj).shortValue());
1094       break;
1095     case LONG:
1096       buffer.putInt(toNumber(obj).intValue());
1097       break;
1098     case MONEY:
1099       writeCurrencyValue(buffer, obj);
1100       break;
1101     case FLOAT:
1102       buffer.putFloat(toNumber(obj).floatValue());
1103       break;
1104     case DOUBLE:
1105       buffer.putDouble(toNumber(obj).doubleValue());
1106       break;
1107     case SHORT_DATE_TIME:
1108       writeDateValue(buffer, obj);
1109       break;
1110     case TEXT:
1111       // apparently text numeric values are also occasionally written as fixed
1112       // length...
1113       int numChars = getLengthInUnits();
1114       // force uncompressed encoding for fixed length text
1115       buffer.put(encodeTextValue(obj, numChars, numChars, true));
1116       break;
1117     case GUID:
1118       writeGUIDValue(buffer, obj);
1119       break;
1120     case NUMERIC:
1121       // yes, that's right, occasionally numeric values are written as fixed
1122       // length...
1123       writeNumericValue(buffer, obj);
1124       break;
1125     case BINARY:
1126     case UNKNOWN_0D:
1127     case UNKNOWN_11:
1128     case COMPLEX_TYPE:
1129       buffer.putInt(toNumber(obj).intValue());
1130       break;
1131     case UNSUPPORTED_FIXEDLEN:
1132       byte[] bytes = toByteArray(obj);
1133       if(bytes.length != getLength()) {
1134         throw new IOException(withErrorContext(
1135                                   "Invalid fixed size binary data, size "
1136                                   + getLength() + ", got " + bytes.length));
1137       }
1138       buffer.put(bytes);
1139       break;
1140     default:
1141       throw new IOException(withErrorContext(
1142                                 "Unsupported data type: " + getType()));
1143     }
1144     return buffer;
1145   }
1146   
1147   /**
1148    * Decodes a compressed or uncompressed text value.
1149    */
1150   String decodeTextValue(byte[] data)
1151     throws IOException
1152   {
1153     // see if data is compressed.  the 0xFF, 0xFE sequence indicates that
1154     // compression is used (sort of, see algorithm below)
1155     boolean isCompressed = ((data.length > 1) &&
1156                             (data[0] == TEXT_COMPRESSION_HEADER[0]) &&
1157                             (data[1] == TEXT_COMPRESSION_HEADER[1]));
1158 
1159     if(isCompressed) {
1160         
1161       // this is a whacky compression combo that switches back and forth
1162       // between compressed/uncompressed using a 0x00 byte (starting in
1163       // compressed mode)
1164       StringBuilder textBuf = new StringBuilder(data.length);
1165       // start after two bytes indicating compression use
1166       int dataStart = TEXT_COMPRESSION_HEADER.length;
1167       int dataEnd = dataStart;
1168       boolean inCompressedMode = true;
1169       while(dataEnd < data.length) {
1170         if(data[dataEnd] == (byte)0x00) {
1171 
1172           // handle current segment
1173           decodeTextSegment(data, dataStart, dataEnd, inCompressedMode,
1174                             textBuf);
1175           inCompressedMode = !inCompressedMode;
1176           ++dataEnd;
1177           dataStart = dataEnd;
1178             
1179         } else {
1180           ++dataEnd;
1181         }
1182       }
1183       // handle last segment
1184       decodeTextSegment(data, dataStart, dataEnd, inCompressedMode, textBuf);
1185 
1186       return textBuf.toString();
1187         
1188     }
1189       
1190     return decodeUncompressedText(data, getCharset());
1191   }
1192 
1193   /**
1194    * Decodes a segnment of a text value into the given buffer according to the
1195    * given status of the segment (compressed/uncompressed).
1196    */
1197   private void decodeTextSegment(byte[] data, int dataStart, int dataEnd,
1198                                  boolean inCompressedMode, 
1199                                  StringBuilder textBuf)
1200   {
1201     if(dataEnd <= dataStart) {
1202       // no data
1203       return;
1204     }
1205     int dataLength = dataEnd - dataStart;
1206 
1207     if(inCompressedMode) {
1208       byte[] tmpData = new byte[dataLength * 2];
1209       int tmpIdx = 0;
1210       for(int i = dataStart; i < dataEnd; ++i) {
1211         tmpData[tmpIdx] = data[i];
1212         tmpIdx += 2;
1213       } 
1214       data = tmpData;
1215       dataStart = 0;
1216       dataLength = data.length;
1217     }
1218 
1219     textBuf.append(decodeUncompressedText(data, dataStart, dataLength,
1220                                           getCharset()));
1221   }
1222 
1223   /**
1224    * @param textBytes bytes of text to decode
1225    * @return the decoded string
1226    */
1227   private static CharBuffer decodeUncompressedText(
1228       byte[] textBytes, int startPos, int length, Charset charset)
1229   {
1230     return charset.decode(ByteBuffer.wrap(textBytes, startPos, length));
1231   }  
1232 
1233   /**
1234    * Encodes a text value, possibly compressing.
1235    */
1236   ByteBuffer encodeTextValue(Object obj, int minChars, int maxChars,
1237                              boolean forceUncompressed)
1238     throws IOException
1239   {
1240     CharSequence text = toCharSequence(obj);
1241     if((text.length() > maxChars) || (text.length() < minChars)) {
1242       throw new IOException(withErrorContext(
1243                             "Text is wrong length for " + getType() +
1244                             " column, max " + maxChars
1245                             + ", min " + minChars + ", got " + text.length()));
1246     }
1247     
1248     // may only compress if column type allows it
1249     if(!forceUncompressed && isCompressedUnicode() &&
1250        (text.length() <= getFormat().MAX_COMPRESSED_UNICODE_SIZE) &&
1251        isUnicodeCompressible(text)) {
1252 
1253       byte[] encodedChars = new byte[TEXT_COMPRESSION_HEADER.length + 
1254                                      text.length()];
1255       encodedChars[0] = TEXT_COMPRESSION_HEADER[0];
1256       encodedChars[1] = TEXT_COMPRESSION_HEADER[1];
1257       for(int i = 0; i < text.length(); ++i) {
1258         encodedChars[i + TEXT_COMPRESSION_HEADER.length] = 
1259           (byte)text.charAt(i);
1260       }
1261       return ByteBuffer.wrap(encodedChars);
1262     }
1263 
1264     return encodeUncompressedText(text, getCharset());
1265   }
1266 
1267   /**
1268    * Returns {@code true} if the given text can be compressed using compressed
1269    * unicode, {@code false} otherwise.
1270    */
1271   private static boolean isUnicodeCompressible(CharSequence text) {
1272     // only attempt to compress > 2 chars (compressing less than 3 chars would
1273     // not result in a space savings due to the 2 byte compression header)
1274     if(text.length() <= TEXT_COMPRESSION_HEADER.length) {
1275       return false;
1276     }
1277     // now, see if it is all compressible characters
1278     for(int i = 0; i < text.length(); ++i) {
1279       char c = text.charAt(i);
1280       if((c < MIN_COMPRESS_CHAR) || (c > MAX_COMPRESS_CHAR)) {
1281         return false;
1282       }
1283     }
1284     return true;
1285   }
1286 
1287   /**
1288    * Constructs a byte containing the flags for this column.
1289    */
1290   private static byte getColumnBitFlags(ColumnBuilder col) {
1291     byte flags = UPDATABLE_FLAG_MASK;
1292     if(!col.isVariableLength()) {
1293       flags |= FIXED_LEN_FLAG_MASK;
1294     }
1295     if(col.isAutoNumber()) {
1296       byte autoNumFlags = 0;
1297       switch(col.getType()) {
1298       case LONG:
1299       case COMPLEX_TYPE:
1300         autoNumFlags = AUTO_NUMBER_FLAG_MASK;
1301         break;
1302       case GUID:
1303         autoNumFlags = AUTO_NUMBER_GUID_FLAG_MASK;
1304         break;
1305       default:
1306         // unknown autonum type
1307       }
1308       flags |= autoNumFlags;
1309     }
1310     if(col.isHyperlink()) {
1311       flags |= HYPERLINK_FLAG_MASK;
1312     }
1313     return flags;
1314   }
1315   
1316   @Override
1317   public String toString() {
1318     ToStringBuilder sb = CustomToStringStyle.builder(this)
1319       .append("name", "(" + _table.getName() + ") " + _name);
1320     byte typeValue = getOriginalDataType();
1321     sb.append("type", "0x" + Integer.toHexString(typeValue) +
1322               " (" + _type + ")")
1323       .append("number", _columnNumber)
1324       .append("length", _columnLength)
1325       .append("variableLength", _variableLength);       
1326     if(_calculated) {
1327       sb.append("calculated", _calculated);
1328     }
1329     if(_type.isTextual()) {
1330       sb.append("compressedUnicode", isCompressedUnicode())
1331         .append("textSortOrder", getTextSortOrder());
1332       if(getTextCodePage() > 0) {
1333         sb.append("textCodePage", getTextCodePage());
1334       }
1335       if(isAppendOnly()) {
1336         sb.append("appendOnly", isAppendOnly());
1337       } 
1338       if(isHyperlink()) {
1339         sb.append("hyperlink", isHyperlink());
1340       } 
1341     }
1342     if(_type.getHasScalePrecision()) {
1343       sb.append("precision", getPrecision())
1344         .append("scale", getScale());
1345     }
1346     if(_autoNumber) {
1347       sb.append("lastAutoNumber", _autoNumberGenerator.getLast());
1348     }
1349     if(getComplexInfo() != null) {
1350       sb.append("complexInfo", getComplexInfo());
1351     }
1352     return sb.toString();
1353   }
1354   
1355   /**
1356    * @param textBytes bytes of text to decode
1357    * @param charset relevant charset
1358    * @return the decoded string
1359    * @usage _advanced_method_
1360    */
1361   public static String decodeUncompressedText(byte[] textBytes, 
1362                                               Charset charset)
1363   {
1364     return decodeUncompressedText(textBytes, 0, textBytes.length, charset)
1365       .toString();
1366   }
1367 
1368   /**
1369    * @param text Text to encode
1370    * @param charset database charset
1371    * @return A buffer with the text encoded
1372    * @usage _advanced_method_
1373    */
1374   public static ByteBuffer encodeUncompressedText(CharSequence text,
1375                                                   Charset charset)
1376   {
1377     CharBuffer cb = ((text instanceof CharBuffer) ? 
1378                      (CharBuffer)text : CharBuffer.wrap(text));
1379     return charset.encode(cb);
1380   }
1381 
1382   
1383   /**
1384    * Orders Columns by column number.
1385    * @usage _general_method_
1386    */
1387   public int compareTo(ColumnImpl other) {
1388     if (_columnNumber > other.getColumnNumber()) {
1389       return 1;
1390     } else if (_columnNumber < other.getColumnNumber()) {
1391       return -1;
1392     } else {
1393       return 0;
1394     }
1395   }
1396   
1397   /**
1398    * @param columns A list of columns in a table definition
1399    * @return The number of variable length columns found in the list
1400    * @usage _advanced_method_
1401    */
1402   public static short countVariableLength(List<ColumnBuilder> columns) {
1403     short rtn = 0;
1404     for (ColumnBuilder col : columns) {
1405       if (col.isVariableLength()) {
1406         rtn++;
1407       }
1408     }
1409     return rtn;
1410   }
1411 
1412   /**
1413    * @return an appropriate BigDecimal representation of the given object.
1414    *         <code>null</code> is returned as 0 and Numbers are converted
1415    *         using their double representation.
1416    */
1417   static BigDecimal toBigDecimal(Object value)
1418   {
1419     if(value == null) {
1420       return BigDecimal.ZERO;
1421     } else if(value instanceof BigDecimal) {
1422       return (BigDecimal)value;
1423     } else if(value instanceof BigInteger) {
1424       return new BigDecimal((BigInteger)value);
1425     } else if(value instanceof Number) {
1426       return new BigDecimal(((Number)value).doubleValue());
1427     }
1428     return new BigDecimal(value.toString());
1429   }
1430 
1431   /**
1432    * @return an appropriate Number representation of the given object.
1433    *         <code>null</code> is returned as 0 and Strings are parsed as
1434    *         Doubles.
1435    */
1436   private static Number toNumber(Object value)
1437   {
1438     if(value == null) {
1439       return BigDecimal.ZERO;
1440     } else if(value instanceof Number) {
1441       return (Number)value;
1442     }
1443     return Double.valueOf(value.toString());
1444   }
1445   
1446   /**
1447    * @return an appropriate CharSequence representation of the given object.
1448    * @usage _advanced_method_
1449    */
1450   public static CharSequence toCharSequence(Object value)
1451     throws IOException
1452   {
1453     if(value == null) {
1454       return null;
1455     } else if(value instanceof CharSequence) {
1456       return (CharSequence)value;
1457     } else if(value instanceof Clob) {
1458       try {
1459         Clob c = (Clob)value;
1460         // note, start pos is 1-based
1461         return c.getSubString(1L, (int)c.length());
1462       } catch(SQLException e) {
1463         throw (IOException)(new IOException(e.getMessage())).initCause(e);
1464       }
1465     } else if(value instanceof Reader) {
1466       char[] buf = new char[8 * 1024];
1467       StringBuilder sout = new StringBuilder();
1468       Reader in = (Reader)value;
1469       int read = 0;
1470       while((read = in.read(buf)) != -1) {
1471         sout.append(buf, 0, read);
1472       }
1473       return sout;
1474     }
1475 
1476     return value.toString();
1477   }
1478 
1479   /**
1480    * @return an appropriate byte[] representation of the given object.
1481    * @usage _advanced_method_
1482    */
1483   public static byte[] toByteArray(Object value)
1484     throws IOException
1485   {
1486     if(value == null) {
1487       return null;
1488     } else if(value instanceof byte[]) {
1489       return (byte[])value;
1490     } else if(value instanceof OleUtil.OleBlobImpl) {
1491       return ((OleUtil.OleBlobImpl)value).getBytes();
1492     } else if(value instanceof Blob) {
1493       try {
1494         Blob b = (Blob)value;
1495         // note, start pos is 1-based
1496         return b.getBytes(1L, (int)b.length());
1497       } catch(SQLException e) {
1498         throw (IOException)(new IOException(e.getMessage())).initCause(e);
1499       }
1500     } else if(value instanceof RawData) {
1501       return ((RawData)value).getBytes();
1502     }
1503 
1504     ByteArrayOutputStream bout = new ByteArrayOutputStream();
1505 
1506     if(value instanceof InputStream) {
1507       ByteUtil.copy((InputStream)value, bout);
1508     } else {
1509       // if all else fails, serialize it
1510       ObjectOutputStream oos = new ObjectOutputStream(bout);
1511       oos.writeObject(value);
1512       oos.close();
1513     }
1514 
1515     return bout.toByteArray();
1516   }
1517 
1518   /**
1519    * Interpret a boolean value (null == false)
1520    * @usage _advanced_method_
1521    */
1522   public static boolean toBooleanValue(Object obj) {
1523     if(obj == null) {
1524       return false;
1525     } else if(obj instanceof Boolean) {
1526       return ((Boolean)obj).booleanValue();
1527     }
1528     return Boolean.parseBoolean(obj.toString());
1529   }
1530   
1531   /**
1532    * Swaps the bytes of the given numeric in place.
1533    */
1534   private static void fixNumericByteOrder(byte[] bytes)
1535   {
1536     // fix endianness of each 4 byte segment
1537     for(int i = 0; i < bytes.length; i+=4) {
1538       ByteUtil.swap4Bytes(bytes, i);
1539     }
1540   }
1541 
1542   /**
1543    * Treat booleans as integers (C-style).
1544    */
1545   protected static Object booleanToInteger(Object obj) {
1546     if (obj instanceof Boolean) {
1547       obj = ((Boolean) obj) ? 1 : 0;
1548     }
1549     return obj;
1550   }
1551 
1552   /**
1553    * Returns a wrapper for raw column data that can be written without
1554    * understanding the data.  Useful for wrapping unparseable data for
1555    * re-writing.
1556    */
1557   public static RawData rawDataWrapper(byte[] bytes) {
1558     return new RawData(bytes);
1559   }
1560 
1561   /**
1562    * Returns {@code true} if the given value is "raw" column data,
1563    * {@code false} otherwise.
1564    * @usage _advanced_method_
1565    */
1566   public static boolean isRawData(Object value) {
1567     return(value instanceof RawData);
1568   }
1569 
1570   /**
1571    * Writes the column definitions into a table definition buffer.
1572    * @param buffer Buffer to write to
1573    */
1574   protected static void writeDefinitions(TableCreator creator, ByteBuffer buffer)
1575     throws IOException
1576   {
1577     // we specifically put the "long variable" values after the normal
1578     // variable length values so that we have a better chance of fitting it
1579     // all (because "long variable" values can go in separate pages)
1580     int longVariableOffset = creator.countNonLongVariableLength();
1581     creator.setColumnOffsets(0, 0, longVariableOffset);
1582 
1583     for (ColumnBuilder col : creator.getColumns()) {
1584       writeDefinition(creator, col, buffer);
1585     }
1586 
1587     for (ColumnBuilder col : creator.getColumns()) {
1588       TableImpl.writeName(buffer, col.getName(), creator.getCharset());
1589     }
1590   }
1591 
1592   protected static void writeDefinition(
1593       TableMutator mutator, ColumnBuilder col, ByteBuffer buffer) 
1594     throws IOException
1595   {
1596     TableMutator.ColumnOffsets colOffsets = mutator.getColumnOffsets();
1597 
1598     buffer.put(col.getType().getValue());
1599     buffer.putInt(TableImpl.MAGIC_TABLE_NUMBER);  //constant magic number
1600     buffer.putShort(col.getColumnNumber());  //Column Number
1601 
1602     if(col.isVariableLength()) {
1603       buffer.putShort(colOffsets.getNextVariableOffset(col));
1604     } else {
1605       buffer.putShort((short) 0);
1606     }
1607 
1608     buffer.putShort(col.getColumnNumber()); //Column Number again
1609 
1610     if(col.getType().isTextual()) {
1611       // this will write 4 bytes (note we don't support writing dbs which
1612       // use the text code page)
1613       writeSortOrder(buffer, col.getTextSortOrder(), mutator.getFormat());
1614     } else {
1615       // note scale/precision not stored for calculated numeric fields
1616       if(col.getType().getHasScalePrecision() && !col.isCalculated()) {
1617         buffer.put(col.getPrecision());  // numeric precision
1618         buffer.put(col.getScale());  // numeric scale
1619       } else {
1620         buffer.put((byte) 0x00); //unused
1621         buffer.put((byte) 0x00); //unused
1622       }
1623       buffer.putShort((short) 0); //Unknown
1624     }
1625 
1626     buffer.put(getColumnBitFlags(col)); // misc col flags
1627 
1628     // note access doesn't seem to allow unicode compression for calced fields
1629     if(col.isCalculated()) {
1630       buffer.put(CALCULATED_EXT_FLAG_MASK);
1631     } else if (col.isCompressedUnicode()) {  //Compressed
1632       buffer.put(COMPRESSED_UNICODE_EXT_FLAG_MASK);
1633     } else {
1634       buffer.put((byte)0);
1635     }
1636 
1637     buffer.putInt(0); //Unknown, but always 0.
1638 
1639     //Offset for fixed length columns
1640     if(col.isVariableLength()) {
1641       buffer.putShort((short) 0);
1642     } else {
1643       buffer.putShort(colOffsets.getNextFixedOffset(col));
1644     }
1645 
1646     if(!col.getType().isLongValue()) {
1647       short length = col.getLength();
1648       if(col.isCalculated()) {
1649         // calced columns have additional value overhead
1650         if(!col.getType().isVariableLength() || 
1651            col.getType().getHasScalePrecision()) {
1652           length = CalculatedColumnUtil.CALC_FIXED_FIELD_LEN; 
1653         } else {
1654           length += CalculatedColumnUtil.CALC_EXTRA_DATA_LEN;
1655         }
1656       }
1657       buffer.putShort(length); //Column length
1658     } else {
1659       buffer.putShort((short)0x0000); // unused
1660     }
1661   }
1662 
1663   protected static void writeColUsageMapDefinitions(
1664       TableCreator creator, ByteBuffer buffer)
1665     throws IOException
1666   {
1667     // write long value column usage map references
1668     for(ColumnBuilder lvalCol : creator.getLongValueColumns()) {
1669       writeColUsageMapDefinition(creator, lvalCol, buffer);
1670     }
1671   }
1672 
1673   protected static void writeColUsageMapDefinition(
1674       TableMutator creator, ColumnBuilder lvalCol, ByteBuffer buffer)
1675     throws IOException
1676   {
1677     TableMutator.ColumnState colState = creator.getColumnState(lvalCol);
1678 
1679     buffer.putShort(lvalCol.getColumnNumber());
1680       
1681     // owned pages umap (both are on same page)
1682     buffer.put(colState.getUmapOwnedRowNumber());
1683     ByteUtil.put3ByteInt(buffer, colState.getUmapPageNumber());
1684     // free space pages umap
1685     buffer.put(colState.getUmapFreeRowNumber());
1686     ByteUtil.put3ByteInt(buffer, colState.getUmapPageNumber());
1687   }
1688 
1689   /**
1690    * Reads the sort order info from the given buffer from the given position.
1691    */
1692   static SortOrder readSortOrder(ByteBuffer buffer, int position,
1693                                  JetFormat format)
1694   {
1695     short value = buffer.getShort(position);
1696     byte version = 0;
1697     if(format.SIZE_SORT_ORDER == 4) {
1698       version = buffer.get(position + 3);
1699     }
1700 
1701     if(value == 0) {
1702       // probably a file we wrote, before handling sort order
1703       return format.DEFAULT_SORT_ORDER;
1704     }
1705     
1706     if(value == GENERAL_SORT_ORDER_VALUE) {
1707       if(version == GENERAL_LEGACY_SORT_ORDER.getVersion()) {
1708         return GENERAL_LEGACY_SORT_ORDER;
1709       }
1710       if(version == GENERAL_SORT_ORDER.getVersion()) {
1711         return GENERAL_SORT_ORDER;
1712       }
1713     }
1714     return new SortOrder(value, version);
1715   }
1716 
1717   /**
1718    * Reads the column cade page info from the given buffer, if supported for
1719    * this db.
1720    */
1721   static short readCodePage(ByteBuffer buffer, int offset, JetFormat format)
1722   {
1723       int cpOffset = format.OFFSET_COLUMN_CODE_PAGE;
1724       return ((cpOffset >= 0) ? buffer.getShort(offset + cpOffset) : 0);
1725   }
1726 
1727   /**
1728    * Read the extra flags field for a column definition.
1729    */
1730   static byte readExtraFlags(ByteBuffer buffer, int offset, JetFormat format)
1731   {
1732     int extFlagsOffset = format.OFFSET_COLUMN_EXT_FLAGS;
1733     return ((extFlagsOffset >= 0) ? buffer.get(offset + extFlagsOffset) : 0);
1734   }
1735   
1736   /**
1737    * Writes the sort order info to the given buffer at the current position.
1738    */
1739   private static void writeSortOrder(ByteBuffer buffer, SortOrder sortOrder,
1740                                      JetFormat format) {
1741     if(sortOrder == null) {
1742       sortOrder = format.DEFAULT_SORT_ORDER;
1743     }
1744     buffer.putShort(sortOrder.getValue());      
1745     if(format.SIZE_SORT_ORDER == 4) {
1746       buffer.put((byte)0x00); // unknown
1747       buffer.put(sortOrder.getVersion());
1748     }
1749   }
1750 
1751   /**
1752    * Returns {@code true} if the value is immutable, {@code false} otherwise.
1753    * This only handles values that are returned from the {@link #read} method.
1754    */
1755   static boolean isImmutableValue(Object value) {
1756     // for now, the only mutable value this class returns is byte[]
1757     return !(value instanceof byte[]);
1758   }
1759 
1760   /**
1761    * Converts the given value to the "internal" representation for the given
1762    * data type.
1763    */
1764   public static Object toInternalValue(DataType dataType, Object value)
1765     throws IOException
1766   {
1767     if(value == null) {
1768       return null;
1769     }
1770 
1771     switch(dataType) {
1772     case BOOLEAN:
1773       return ((value instanceof Boolean) ? value : toBooleanValue(value));
1774     case BYTE:
1775       return ((value instanceof Byte) ? value : toNumber(value).byteValue());
1776     case INT:
1777       return ((value instanceof Short) ? value : 
1778               toNumber(value).shortValue());
1779     case LONG:
1780       return ((value instanceof Integer) ? value : 
1781               toNumber(value).intValue());
1782     case MONEY:
1783       return toBigDecimal(value);
1784     case FLOAT:
1785       return ((value instanceof Float) ? value : 
1786               toNumber(value).floatValue());
1787     case DOUBLE:
1788       return ((value instanceof Double) ? value : 
1789               toNumber(value).doubleValue());
1790     case SHORT_DATE_TIME:
1791       return ((value instanceof DateExt) ? value :
1792               new Date(toDateLong(value)));
1793     case TEXT:
1794     case MEMO:
1795     case GUID:
1796       return ((value instanceof String) ? value : 
1797               toCharSequence(value).toString());
1798     case NUMERIC:
1799       return toBigDecimal(value);
1800     case COMPLEX_TYPE:
1801       // leave alone for now?
1802       return value;
1803     default:
1804       // some variation of binary data
1805       return toByteArray(value);
1806     }
1807   }
1808 
1809   String withErrorContext(String msg) {
1810     return withErrorContext(msg, getDatabase(), getTable().getName(), getName());
1811   }
1812 
1813   private static String withErrorContext(
1814       String msg, DatabaseImpl db, String tableName, String colName) {
1815     return msg + " (Db=" + db.getName() + ";Table=" + tableName + ";Column=" + 
1816       colName + ")";
1817   }
1818 
1819   /**
1820    * Date subclass which stashes the original date bits, in case we attempt to
1821    * re-write the value (will not lose precision).
1822    */
1823   private static final class DateExt extends Date
1824   {
1825     private static final long serialVersionUID = 0L;
1826 
1827     /** cached bits of the original date value */
1828     private transient final long _dateBits;
1829 
1830     private DateExt(long time, long dateBits) {
1831       super(time);
1832       _dateBits = dateBits;
1833     }
1834 
1835     public long getDateBits() {
1836       return _dateBits;
1837     }
1838     
1839     private Object writeReplace() throws ObjectStreamException {
1840       // if we are going to serialize this Date, convert it back to a normal
1841       // Date (in case it is restored outside of the context of jackcess)
1842       return new Date(super.getTime());
1843     }
1844   }
1845 
1846   /**
1847    * Wrapper for raw column data which can be re-written.
1848    */
1849   private static class RawData implements Serializable
1850   {
1851     private static final long serialVersionUID = 0L;
1852 
1853     private final byte[] _bytes;
1854 
1855     private RawData(byte[] bytes) {
1856       _bytes = bytes;
1857     }
1858 
1859     private byte[] getBytes() {
1860       return _bytes;
1861     }
1862 
1863     @Override
1864     public String toString() {
1865       return CustomToStringStyle.valueBuilder(this)
1866         .append(null, getBytes())
1867         .toString();
1868     }
1869 
1870     private Object writeReplace() throws ObjectStreamException {
1871       // if we are going to serialize this, convert it back to a normal
1872       // byte[] (in case it is restored outside of the context of jackcess)
1873       return getBytes();
1874     }
1875   }
1876 
1877   /**
1878    * Base class for the supported autonumber types.
1879    * @usage _advanced_class_
1880    */
1881   public abstract class AutoNumberGenerator
1882   {
1883     protected AutoNumberGenerator() {}
1884 
1885     /**
1886      * Returns the last autonumber generated by this generator.  Only valid
1887      * after a call to {@link Table#addRow}, otherwise undefined.
1888      */
1889     public abstract Object getLast();
1890 
1891     /**
1892      * Returns the next autonumber for this generator.
1893      * <p>
1894      * <i>Warning, calling this externally will result in this value being
1895      * "lost" for the table.</i>
1896      */
1897     public abstract Object getNext(TableImpl.WriteRowState writeRowState);
1898 
1899     /**
1900      * Returns a valid autonumber for this generator.
1901      * <p>
1902      * <i>Warning, calling this externally may result in this value being
1903      * "lost" for the table.</i>
1904      */
1905     public abstract Object handleInsert(
1906         TableImpl.WriteRowState writeRowState, Object inRowValue) 
1907       throws IOException;
1908 
1909     /**
1910      * Restores a previous autonumber generated by this generator.
1911      */
1912     public abstract void restoreLast(Object last);
1913     
1914     /**
1915      * Returns the type of values generated by this generator.
1916      */
1917     public abstract DataType getType();
1918   }
1919 
1920   private final class LongAutoNumberGenerator extends AutoNumberGenerator
1921   {
1922     private LongAutoNumberGenerator() {}
1923 
1924     @Override
1925     public Object getLast() {
1926       // the table stores the last long autonumber used
1927       return getTable().getLastLongAutoNumber();
1928     }
1929 
1930     @Override
1931     public Object getNext(TableImpl.WriteRowState writeRowState) {
1932       // the table stores the last long autonumber used
1933       return getTable().getNextLongAutoNumber();
1934     }
1935 
1936     @Override
1937     public Object handleInsert(TableImpl.WriteRowState writeRowState,
1938                                Object inRowValue) 
1939       throws IOException
1940     {
1941       int inAutoNum = toNumber(inRowValue).intValue();
1942       if(inAutoNum <= INVALID_AUTO_NUMBER && !getTable().isAllowAutoNumberInsert()) {
1943         throw new IOException(withErrorContext(
1944                 "Invalid auto number value " + inAutoNum));
1945       }
1946       // the table stores the last long autonumber used
1947       getTable().adjustLongAutoNumber(inAutoNum);
1948       return inAutoNum;
1949     }
1950 
1951     @Override
1952     public void restoreLast(Object last) {
1953       if(last instanceof Integer) {
1954         getTable().restoreLastLongAutoNumber((Integer)last);
1955       }
1956     }
1957     
1958     @Override
1959     public DataType getType() {
1960       return DataType.LONG;
1961     }
1962   }
1963 
1964   private final class GuidAutoNumberGenerator extends AutoNumberGenerator
1965   {
1966     private Object _lastAutoNumber;
1967 
1968     private GuidAutoNumberGenerator() {}
1969 
1970     @Override
1971     public Object getLast() {
1972       return _lastAutoNumber;
1973     }
1974 
1975     @Override
1976     public Object getNext(TableImpl.WriteRowState writeRowState) {
1977       // format guids consistently w/ Column.readGUIDValue()
1978       _lastAutoNumber = "{" + UUID.randomUUID() + "}";
1979       return _lastAutoNumber;
1980     }
1981 
1982     @Override
1983     public Object handleInsert(TableImpl.WriteRowState writeRowState,
1984                                Object inRowValue) 
1985       throws IOException
1986     {
1987       _lastAutoNumber = toCharSequence(inRowValue);
1988       return _lastAutoNumber;
1989     }
1990 
1991     @Override
1992     public void restoreLast(Object last) {
1993       _lastAutoNumber = null;
1994     }
1995     
1996     @Override
1997     public DataType getType() {
1998       return DataType.GUID;
1999     }
2000   }
2001 
2002   private final class ComplexTypeAutoNumberGenerator extends AutoNumberGenerator
2003   {
2004     private ComplexTypeAutoNumberGenerator() {}
2005 
2006     @Override
2007     public Object getLast() {
2008       // the table stores the last ComplexType autonumber used
2009       return getTable().getLastComplexTypeAutoNumber();
2010     }
2011 
2012     @Override
2013     public Object getNext(TableImpl.WriteRowState writeRowState) {
2014       // same value is shared across all ComplexType values in a row
2015       int nextComplexAutoNum = writeRowState.getComplexAutoNumber();
2016       if(nextComplexAutoNum <= INVALID_AUTO_NUMBER) {
2017         // the table stores the last ComplexType autonumber used
2018         nextComplexAutoNum = getTable().getNextComplexTypeAutoNumber();
2019         writeRowState.setComplexAutoNumber(nextComplexAutoNum);
2020       }
2021       return new ComplexValueForeignKeyImpl(ColumnImpl.this, 
2022                                             nextComplexAutoNum);
2023     }
2024 
2025     @Override
2026     public Object handleInsert(TableImpl.WriteRowState writeRowState,
2027                                Object inRowValue) 
2028       throws IOException
2029     {
2030       ComplexValueForeignKey inComplexFK = null;
2031       if(inRowValue instanceof ComplexValueForeignKey) {
2032         inComplexFK = (ComplexValueForeignKey)inRowValue;
2033       } else {
2034         inComplexFK = new ComplexValueForeignKeyImpl(
2035             ColumnImpl.this, toNumber(inRowValue).intValue());
2036       }
2037 
2038       if(inComplexFK.getColumn() != ColumnImpl.this) {
2039         throw new IOException(withErrorContext(
2040                 "Wrong column for complex value foreign key, found " +
2041                 inComplexFK.getColumn().getName()));
2042       }
2043       if(inComplexFK.get() < 1) {
2044         throw new IOException(withErrorContext(
2045                 "Invalid complex value foreign key value " + inComplexFK.get()));
2046       }
2047       // same value is shared across all ComplexType values in a row
2048       int prevRowValue = writeRowState.getComplexAutoNumber();
2049       if(prevRowValue <= INVALID_AUTO_NUMBER) {
2050         writeRowState.setComplexAutoNumber(inComplexFK.get());
2051       } else if(prevRowValue != inComplexFK.get()) {
2052         throw new IOException(withErrorContext(
2053                 "Inconsistent complex value foreign key values: found " +
2054                 prevRowValue + ", given " + inComplexFK));
2055       }
2056 
2057       // the table stores the last ComplexType autonumber used
2058       getTable().adjustComplexTypeAutoNumber(inComplexFK.get());
2059 
2060       return inComplexFK;
2061     }
2062 
2063     @Override
2064     public void restoreLast(Object last) {
2065       if(last instanceof ComplexValueForeignKey) {
2066         getTable().restoreLastComplexTypeAutoNumber(
2067             ((ComplexValueForeignKey)last).get());
2068       }
2069     }
2070     
2071     @Override
2072     public DataType getType() {
2073       return DataType.COMPLEX_TYPE;
2074     }
2075   }
2076   
2077   private final class UnsupportedAutoNumberGenerator extends AutoNumberGenerator
2078   {
2079     private final DataType _genType;
2080     
2081     private UnsupportedAutoNumberGenerator(DataType genType) {
2082       _genType = genType;
2083     }
2084     
2085     @Override
2086     public Object getLast() {
2087       return null;
2088     }
2089 
2090     @Override
2091     public Object getNext(TableImpl.WriteRowState writeRowState) {
2092       throw new UnsupportedOperationException();
2093     }
2094 
2095     @Override
2096     public Object handleInsert(TableImpl.WriteRowState writeRowState,
2097                                Object inRowValue) {
2098       throw new UnsupportedOperationException();
2099     }
2100 
2101     @Override
2102     public void restoreLast(Object last) {
2103       throw new UnsupportedOperationException();
2104     }
2105     
2106     @Override
2107     public DataType getType() {
2108       return _genType;
2109     }
2110   }
2111 
2112   
2113   /**
2114    * Information about the sort order (collation) for a textual column.
2115    * @usage _intermediate_class_
2116    */
2117   public static final class SortOrder
2118   {
2119     private final short _value;
2120     private final byte _version;
2121     
2122     public SortOrder(short value, byte version) {
2123       _value = value;
2124       _version = version;
2125     }
2126 
2127     public short getValue() {
2128       return _value;
2129     }
2130 
2131     public byte getVersion() {
2132       return _version;
2133     }
2134 
2135     @Override
2136     public int hashCode() {
2137       return _value;
2138     }
2139 
2140     @Override
2141     public boolean equals(Object o) {
2142       return ((this == o) ||
2143               ((o != null) && (getClass() == o.getClass()) &&
2144                (_value == ((SortOrder)o)._value) &&
2145                (_version == ((SortOrder)o)._version)));
2146     }
2147 
2148     @Override
2149     public String toString() {
2150       return CustomToStringStyle.valueBuilder(this)
2151         .append(null, _value + "(" + _version + ")")
2152         .toString();
2153     }
2154   }
2155 
2156   /**
2157    * Utility struct for passing params through ColumnImpl constructors.
2158    */
2159   static final class InitArgs
2160   {
2161     public final TableImpl table;
2162     public final ByteBuffer buffer;
2163     public final int offset;
2164     public final String name;
2165     public final int displayIndex;
2166     public final byte colType;
2167     public final byte flags;
2168     public final byte extFlags;
2169     public DataType type;
2170 
2171     InitArgs(TableImpl table, ByteBuffer buffer, int offset, String name,
2172              int displayIndex) {
2173       this.table = table;
2174       this.buffer = buffer;
2175       this.offset = offset;
2176       this.name = name;
2177       this.displayIndex = displayIndex;
2178       
2179       this.colType = buffer.get(offset + table.getFormat().OFFSET_COLUMN_TYPE);
2180       this.flags = buffer.get(offset + table.getFormat().OFFSET_COLUMN_FLAGS);
2181       this.extFlags = readExtraFlags(buffer, offset, table.getFormat());
2182     }
2183   }
2184 }