View Javadoc
1   /*
2   Copyright (c) 2005 Health Market Science, Inc.
3   
4   Licensed under the Apache License, Version 2.0 (the "License");
5   you may not use this file except in compliance with the License.
6   You may obtain a copy of the License at
7   
8       http://www.apache.org/licenses/LICENSE-2.0
9   
10  Unless required by applicable law or agreed to in writing, software
11  distributed under the License is distributed on an "AS IS" BASIS,
12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  See the License for the specific language governing permissions and
14  limitations under the License.
15  */
16  
17  package com.healthmarketscience.jackcess.impl;
18  
19  import java.io.ByteArrayOutputStream;
20  import java.io.IOException;
21  import java.io.InputStream;
22  import java.io.ObjectOutputStream;
23  import java.io.ObjectStreamException;
24  import java.io.Reader;
25  import java.io.Serializable;
26  import java.math.BigDecimal;
27  import java.math.BigInteger;
28  import java.nio.ByteBuffer;
29  import java.nio.ByteOrder;
30  import java.nio.CharBuffer;
31  import java.nio.charset.Charset;
32  import java.sql.Blob;
33  import java.sql.Clob;
34  import java.sql.SQLException;
35  import java.util.Calendar;
36  import java.util.Collection;
37  import java.util.Date;
38  import java.util.List;
39  import java.util.Map;
40  import java.util.UUID;
41  import java.util.regex.Matcher;
42  import java.util.regex.Pattern;
43  
44  import com.healthmarketscience.jackcess.Column;
45  import com.healthmarketscience.jackcess.ColumnBuilder;
46  import com.healthmarketscience.jackcess.DataType;
47  import com.healthmarketscience.jackcess.PropertyMap;
48  import com.healthmarketscience.jackcess.Table;
49  import com.healthmarketscience.jackcess.complex.ComplexColumnInfo;
50  import com.healthmarketscience.jackcess.complex.ComplexValue;
51  import com.healthmarketscience.jackcess.complex.ComplexValueForeignKey;
52  import com.healthmarketscience.jackcess.impl.complex.ComplexValueForeignKeyImpl;
53  import com.healthmarketscience.jackcess.util.ColumnValidator;
54  import com.healthmarketscience.jackcess.util.SimpleColumnValidator;
55  import org.apache.commons.lang.builder.ToStringBuilder;
56  import org.apache.commons.logging.Log;
57  import org.apache.commons.logging.LogFactory;
58  
59  /**
60   * Access database column definition
61   * @author Tim McCune
62   * @usage _intermediate_class_
63   */
64  public class ColumnImpl implements Column, Comparable<ColumnImpl> {
65    
66    protected static final Log LOG = LogFactory.getLog(ColumnImpl.class);
67    
68    /**
69     * Placeholder object for adding rows which indicates that the caller wants
70     * the RowId of the new row.  Must be added as an extra value at the end of
71     * the row values array.
72     * @see TableImpl#asRowWithRowId
73     * @usage _intermediate_field_
74     */
75    public static final Object RETURN_ROW_ID = "<RETURN_ROW_ID>";
76  
77    /**
78     * Access stores numeric dates in days.  Java stores them in milliseconds.
79     */
80    private static final long MILLISECONDS_PER_DAY =
81      (24L * 60L * 60L * 1000L);
82  
83    /**
84     * Access starts counting dates at Dec 30, 1899 (note, this strange date
85     * seems to be caused by MS compatibility with Lotus-1-2-3 and incorrect
86     * leap years).  Java starts counting at Jan 1, 1970.  This is the # of
87     * millis between them for conversion.
88     */
89    static final long MILLIS_BETWEEN_EPOCH_AND_1900 =
90      25569L * MILLISECONDS_PER_DAY;
91    
92    /**
93     * mask for the fixed len bit
94     * @usage _advanced_field_
95     */
96    public static final byte FIXED_LEN_FLAG_MASK = (byte)0x01;
97    
98    /**
99     * mask for the auto number bit
100    * @usage _advanced_field_
101    */
102   public static final byte AUTO_NUMBER_FLAG_MASK = (byte)0x04;
103   
104   /**
105    * mask for the auto number guid bit
106    * @usage _advanced_field_
107    */
108   public static final byte AUTO_NUMBER_GUID_FLAG_MASK = (byte)0x40;
109   
110   /**
111    * mask for the hyperlink bit (on memo types)
112    * @usage _advanced_field_
113    */
114   public static final byte HYPERLINK_FLAG_MASK = (byte)0x80;
115   
116   /**
117    * mask for the "is updatable" field bit
118    * @usage _advanced_field_
119    */
120   public static final byte UPDATABLE_FLAG_MASK = (byte)0x02;
121 
122   // some other flags?
123   // 0x10: replication related field (or hidden?)
124 
125   protected static final byte COMPRESSED_UNICODE_EXT_FLAG_MASK = (byte)0x01;
126   private static final byte CALCULATED_EXT_FLAG_MASK = (byte)0xC0;
127 
128   static final byte NUMERIC_NEGATIVE_BYTE = (byte)0x80;
129 
130   /** the value for the "general" sort order */
131   private static final short GENERAL_SORT_ORDER_VALUE = 1033;
132 
133   /**
134    * the "general" text sort order, legacy version (access 2000-2007)
135    * @usage _intermediate_field_
136    */
137   public static final SortOrder GENERAL_LEGACY_SORT_ORDER =
138     new SortOrder(GENERAL_SORT_ORDER_VALUE, (byte)0);
139 
140   /**
141    * the "general" text sort order, latest version (access 2010+)
142    * @usage _intermediate_field_
143    */
144   public static final SortOrder GENERAL_SORT_ORDER = 
145     new SortOrder(GENERAL_SORT_ORDER_VALUE, (byte)1);
146 
147   /** pattern matching textual guid strings (allows for optional surrounding
148       '{' and '}') */
149   private static final Pattern GUID_PATTERN = Pattern.compile("\\s*[{]?([\\p{XDigit}]{8})-([\\p{XDigit}]{4})-([\\p{XDigit}]{4})-([\\p{XDigit}]{4})-([\\p{XDigit}]{12})[}]?\\s*");
150 
151   /** header used to indicate unicode text compression */
152   private static final byte[] TEXT_COMPRESSION_HEADER = 
153   { (byte)0xFF, (byte)0XFE };
154   private static final char MIN_COMPRESS_CHAR = 1;
155   private static final char MAX_COMPRESS_CHAR = 0xFF;
156 
157   /** auto numbers must be > 0 */
158   static final int INVALID_AUTO_NUMBER = 0;
159 
160 
161   /** owning table */
162   private final TableImpl _table;
163   /** Whether or not the column is of variable length */
164   private final boolean _variableLength;
165   /** Whether or not the column is an autonumber column */
166   private final boolean _autoNumber;
167   /** Whether or not the column is a calculated column */
168   private final boolean _calculated;
169   /** Data type */
170   private final DataType _type;
171   /** Maximum column length */
172   private final short _columnLength;
173   /** 0-based column number */
174   private final short _columnNumber;
175   /** index of the data for this column within a list of row data */
176   private int _columnIndex;
177   /** display index of the data for this column */
178   private final int _displayIndex;
179   /** Column name */
180   private final String _name;
181   /** the offset of the fixed data in the row */
182   private final int _fixedDataOffset;
183   /** the index of the variable length data in the var len offset table */
184   private final int _varLenTableIndex;
185   /** the auto number generator for this column (if autonumber column) */
186   private final AutoNumberGenerator _autoNumberGenerator;
187   /** properties for this column, if any */
188   private PropertyMap _props;  
189   /** Validator for writing new values */
190   private ColumnValidator _validator = SimpleColumnValidator.INSTANCE;
191   
192   /**
193    * @usage _advanced_method_
194    */
195   protected ColumnImpl(TableImpl table, String name, DataType type,
196                        int colNumber, int fixedOffset, int varLenIndex) {
197     _table = table;
198     _name = name;
199     _type = type;
200 
201     if(!_type.isVariableLength()) {
202       _columnLength = (short)type.getFixedSize();
203     } else {
204       _columnLength = (short)type.getMaxSize();
205     }
206     _variableLength = type.isVariableLength();
207     _autoNumber = false;
208     _calculated = false;
209     _autoNumberGenerator = null;
210     _columnNumber = (short)colNumber;
211     _columnIndex = colNumber;
212     _displayIndex = colNumber;
213     _fixedDataOffset = fixedOffset;
214     _varLenTableIndex = varLenIndex;
215   }
216     
217   /**
218    * Read a column definition in from a buffer
219    * @usage _advanced_method_
220    */
221   ColumnImpl(InitArgs args)
222     throws IOException
223   {
224     _table = args.table;
225     _name = args.name;
226     _displayIndex = args.displayIndex;
227     _type = args.type;
228     
229     _columnNumber = args.buffer.getShort(
230         args.offset + getFormat().OFFSET_COLUMN_NUMBER);
231     _columnLength = args.buffer.getShort(
232         args.offset + getFormat().OFFSET_COLUMN_LENGTH);
233     
234     _variableLength = ((args.flags & FIXED_LEN_FLAG_MASK) == 0);
235     _autoNumber = ((args.flags & 
236                     (AUTO_NUMBER_FLAG_MASK | AUTO_NUMBER_GUID_FLAG_MASK)) != 0);
237     _calculated = ((args.extFlags & CALCULATED_EXT_FLAG_MASK) != 0);
238     
239     _autoNumberGenerator = createAutoNumberGenerator();
240     
241     if(_variableLength) {
242       _varLenTableIndex = args.buffer.getShort(
243           args.offset + getFormat().OFFSET_COLUMN_VARIABLE_TABLE_INDEX);
244       _fixedDataOffset = 0;
245     } else {
246       _fixedDataOffset = args.buffer.getShort(
247           args.offset + getFormat().OFFSET_COLUMN_FIXED_DATA_OFFSET);
248       _varLenTableIndex = 0;
249     }
250   }
251   
252   /**
253    * Creates the appropriate ColumnImpl class and reads a column definition in
254    * from a buffer
255    * @param table owning table
256    * @param buffer Buffer containing column definition
257    * @param offset Offset in the buffer at which the column definition starts
258    * @usage _advanced_method_
259    */
260   public static ColumnImpl create(TableImpl table, ByteBuffer buffer,
261                                   int offset, String name, int displayIndex)
262     throws IOException
263   {
264     InitArgs args = new InitArgs(table, buffer, offset, name, displayIndex);
265 
266     boolean calculated = ((args.extFlags & CALCULATED_EXT_FLAG_MASK) != 0);
267     byte colType = args.colType;
268     if(calculated) {
269       // "real" data type is in the "result type" property
270       PropertyMap colProps = table.getPropertyMaps().get(name);
271       Byte resultType = (Byte)colProps.getValue(PropertyMap.RESULT_TYPE_PROP);
272       if(resultType != null) {
273         colType = resultType;
274       }
275     }
276     
277     try {
278       args.type = DataType.fromByte(colType);
279     } catch(IOException e) {
280       LOG.warn(withErrorContext("Unsupported column type " + colType,
281                                 table.getDatabase(), table.getName(), name));
282       boolean variableLength = ((args.flags & FIXED_LEN_FLAG_MASK) == 0);
283       args.type = (variableLength ? DataType.UNSUPPORTED_VARLEN :
284                    DataType.UNSUPPORTED_FIXEDLEN);
285       return new UnsupportedColumnImpl(args);
286     }
287 
288     if(calculated) {
289       return CalculatedColumnUtil.create(args);
290     }
291     
292     switch(args.type) {
293     case TEXT:
294       return new TextColumnImpl(args);
295     case MEMO:
296       return new MemoColumnImpl(args);
297     case COMPLEX_TYPE:
298       return new ComplexColumnImpl(args);
299     default:
300       // fall through
301     }
302 
303     if(args.type.getHasScalePrecision()) {
304       return new NumericColumnImpl(args);
305     }
306     if(args.type.isLongValue()) {
307       return new LongValueColumnImpl(args);
308     }
309     
310     return new ColumnImpl(args);
311   }
312 
313   /**
314    * Sets the usage maps for this column.
315    */
316   void setUsageMaps(UsageMap ownedPages, UsageMap freeSpacePages) {
317     // base does nothing
318   }
319 
320   void collectUsageMapPages(Collection<Integer> pages) {
321     // base does nothing
322   }
323     
324   /**
325    * Secondary column initialization after the table is fully loaded.
326    */
327   void postTableLoadInit() throws IOException {
328     // base does nothing
329   }
330 
331   public TableImpl getTable() {
332     return _table;
333   }
334 
335   public DatabaseImpl getDatabase() {       
336     return getTable().getDatabase();
337   }
338   
339   /**
340    * @usage _advanced_method_
341    */
342   public JetFormat getFormat() {
343     return getDatabase().getFormat();
344   }
345 
346   /**
347    * @usage _advanced_method_
348    */
349   public PageChannel getPageChannel() {
350     return getDatabase().getPageChannel();
351   }
352   
353   public String getName() {
354     return _name;
355   }
356   
357   public boolean isVariableLength() {
358     return _variableLength;
359   }
360   
361   public boolean isAutoNumber() {
362     return _autoNumber;
363   }
364 
365   /**
366    * @usage _advanced_method_
367    */
368   public short getColumnNumber() {
369     return _columnNumber;
370   }
371 
372   public int getColumnIndex() {
373     return _columnIndex;
374   }
375 
376   /**
377    * @usage _advanced_method_
378    */
379   public void setColumnIndex(int newColumnIndex) {
380     _columnIndex = newColumnIndex;
381   }
382   
383   /**
384    * @usage _advanced_method_
385    */
386   public int getDisplayIndex() {
387     return _displayIndex;
388   }
389 
390   public DataType getType() {
391     return _type;
392   }
393   
394   public int getSQLType() throws SQLException {
395     return _type.getSQLType();
396   }
397   
398   public boolean isCompressedUnicode() {
399     return false;
400   }
401 
402   public byte getPrecision() {
403     return (byte)getType().getDefaultPrecision();
404   }
405   
406   public byte getScale() {
407     return (byte)getType().getDefaultScale();
408   }
409 
410   /**
411    * @usage _intermediate_method_
412    */
413   public SortOrder getTextSortOrder() {
414     return null;
415   }
416 
417   /**
418    * @usage _intermediate_method_
419    */
420   public short getTextCodePage() {
421     return 0;
422   }
423 
424   public short getLength() {
425     return _columnLength;
426   }
427 
428   public short getLengthInUnits() {
429     return (short)getType().toUnitSize(getLength());
430   }
431 
432   public boolean isCalculated() {
433     return _calculated;
434   }
435   
436   /**
437    * @usage _advanced_method_
438    */
439   public int getVarLenTableIndex() {
440     return _varLenTableIndex;
441   }
442   
443   /**
444    * @usage _advanced_method_
445    */
446   public int getFixedDataOffset() {
447     return _fixedDataOffset;
448   }
449 
450   protected Charset getCharset() {
451     return getDatabase().getCharset();
452   }
453 
454   protected Calendar getCalendar() {
455     return getDatabase().getCalendar();
456   }
457 
458   public boolean isAppendOnly() {
459     return (getVersionHistoryColumn() != null);
460   }
461   
462   public ColumnImpl getVersionHistoryColumn() {
463     return null;
464   }
465 
466   /**
467    * Returns the number of database pages owned by this column.
468    * @usage _intermediate_method_
469    */
470   public int getOwnedPageCount() {
471     return 0;
472   }
473 
474   /**
475    * @usage _advanced_method_
476    */
477   public void setVersionHistoryColumn(ColumnImpl versionHistoryCol) {
478     throw new UnsupportedOperationException();
479   }
480 
481   public boolean isHyperlink() {
482     return false;
483   }
484   
485   public ComplexColumnInfo<? extends ComplexValue> getComplexInfo() {
486     return null;
487   }
488 
489   public ColumnValidator getColumnValidator() {
490     return _validator;
491   }
492   
493   public void setColumnValidator(ColumnValidator newValidator) {
494     
495     if(isAutoNumber()) {
496       // cannot set autonumber validator (autonumber values are controlled
497       // internally)
498       if(newValidator != null) {
499         throw new IllegalArgumentException(withErrorContext(
500                 "Cannot set ColumnValidator for autonumber columns"));
501       }
502       // just leave default validator instance alone
503       return;
504     }
505     
506     if(newValidator == null) {
507       newValidator = getDatabase().getColumnValidatorFactory()
508         .createValidator(this);
509       if(newValidator == null) {
510         newValidator = SimpleColumnValidator.INSTANCE;
511       }
512     }
513     _validator = newValidator;
514   }
515   
516   byte getOriginalDataType() {
517     return _type.getValue();
518   }
519   
520   private AutoNumberGenerator createAutoNumberGenerator() {
521     if(!_autoNumber || (_type == null)) {
522       return null;
523     }
524 
525     switch(_type) {
526     case LONG:
527       return new LongAutoNumberGenerator();
528     case GUID:
529       return new GuidAutoNumberGenerator();
530     case COMPLEX_TYPE:
531       return new ComplexTypeAutoNumberGenerator();
532     default:
533       LOG.warn(withErrorContext("Unknown auto number column type " + _type));
534       return new UnsupportedAutoNumberGenerator(_type);
535     }
536   }
537 
538   /**
539    * Returns the AutoNumberGenerator for this column if this is an autonumber
540    * column, {@code null} otherwise.
541    * @usage _advanced_method_
542    */
543   public AutoNumberGenerator getAutoNumberGenerator() {
544     return _autoNumberGenerator;
545   }
546 
547   public PropertyMap getProperties() throws IOException {
548     if(_props == null) {
549       _props = getTable().getPropertyMaps().get(getName());
550     }
551     return _props;
552   }
553   
554   public Object setRowValue(Object[] rowArray, Object value) {
555     rowArray[_columnIndex] = value;
556     return value;
557   }
558   
559   public Object setRowValue(Map<String,Object> rowMap, Object value) {
560     rowMap.put(_name, value);
561     return value;
562   }
563   
564   public Object getRowValue(Object[] rowArray) {
565     return rowArray[_columnIndex];
566   }
567   
568   public Object getRowValue(Map<String,?> rowMap) {
569     return rowMap.get(_name);
570   }
571 
572   public boolean storeInNullMask() {
573     return (getType() == DataType.BOOLEAN);
574   }
575   
576   public boolean writeToNullMask(Object value) {
577     return toBooleanValue(value);
578   }
579 
580   public Object readFromNullMask(boolean isNull) {
581     return Boolean.valueOf(!isNull);
582   }
583 
584   /**
585    * Deserialize a raw byte value for this column into an Object
586    * @param data The raw byte value
587    * @return The deserialized Object
588    * @usage _advanced_method_
589    */
590   public Object read(byte[] data) throws IOException {
591     return read(data, PageChannel.DEFAULT_BYTE_ORDER);
592   }
593   
594   /**
595    * Deserialize a raw byte value for this column into an Object
596    * @param data The raw byte value
597    * @param order Byte order in which the raw value is stored
598    * @return The deserialized Object
599    * @usage _advanced_method_
600    */  
601   public Object read(byte[] data, ByteOrder order) throws IOException {
602     ByteBuffer buffer = ByteBuffer.wrap(data).order(order);
603 
604     switch(getType()) {
605     case BOOLEAN:
606       throw new IOException(withErrorContext("Tried to read a boolean from data instead of null mask."));
607     case BYTE:
608       return Byte.valueOf(buffer.get());
609     case INT:
610       return Short.valueOf(buffer.getShort());
611     case LONG:
612       return Integer.valueOf(buffer.getInt());
613     case DOUBLE:
614       return Double.valueOf(buffer.getDouble());
615     case FLOAT:
616       return Float.valueOf(buffer.getFloat());
617     case SHORT_DATE_TIME:
618       return readDateValue(buffer);
619     case BINARY:
620       return data;
621     case TEXT:
622       return decodeTextValue(data);
623     case MONEY:
624       return readCurrencyValue(buffer);
625     case NUMERIC:
626       return readNumericValue(buffer);
627     case GUID:
628       return readGUIDValue(buffer, order);
629     case UNKNOWN_0D:
630     case UNKNOWN_11:
631       // treat like "binary" data
632       return data;
633     case COMPLEX_TYPE:
634       return new ComplexValueForeignKeyImpl(this, buffer.getInt());
635     case BIG_INT:
636       return Long.valueOf(buffer.getLong());
637     default:
638       throw new IOException(withErrorContext("Unrecognized data type: " + _type));
639     }
640   }
641 
642   /**
643    * Decodes "Currency" values.
644    * 
645    * @param buffer Column value that points to currency data
646    * @return BigDecimal representing the monetary value
647    * @throws IOException if the value cannot be parsed 
648    */
649   private BigDecimal readCurrencyValue(ByteBuffer buffer)
650     throws IOException
651   {
652     if(buffer.remaining() != 8) {
653       throw new IOException(withErrorContext("Invalid money value"));
654     }
655     
656     return new BigDecimal(BigInteger.valueOf(buffer.getLong(0)), 4);
657   }
658 
659   /**
660    * Writes "Currency" values.
661    */
662   private void writeCurrencyValue(ByteBuffer buffer, Object value)
663     throws IOException
664   {
665     Object inValue = value;
666     try {
667       BigDecimal decVal = toBigDecimal(value);
668       inValue = decVal;
669 
670       // adjust scale (will cause the an ArithmeticException if number has too
671       // many decimal places)
672       decVal = decVal.setScale(4);
673     
674       // now, remove scale and convert to long (this will throw if the value is
675       // too big)
676       buffer.putLong(decVal.movePointRight(4).longValueExact());
677     } catch(ArithmeticException e) {
678       throw (IOException)
679         new IOException(withErrorContext(
680                 "Currency value '" + inValue + "' out of range"))
681         .initCause(e);
682     }
683   }
684 
685   /**
686    * Decodes a NUMERIC field.
687    */
688   private BigDecimal readNumericValue(ByteBuffer buffer)
689   {
690     boolean negate = (buffer.get() != 0);
691 
692     byte[] tmpArr = ByteUtil.getBytes(buffer, 16);
693 
694     if(buffer.order() != ByteOrder.BIG_ENDIAN) {
695       fixNumericByteOrder(tmpArr);
696     }
697 
698     return toBigDecimal(tmpArr, negate, getScale());
699   }
700 
701   static BigDecimal toBigDecimal(byte[] bytes, boolean negate, int scale)
702   {
703     if((bytes[0] & 0x80) != 0) {
704       // the data is effectively unsigned, but the BigInteger handles it as
705       // signed twos complement.  we need to add an extra byte to the input so
706       // that it will be treated as unsigned
707       bytes = ByteUtil.copyOf(bytes, 0, bytes.length + 1, 1);
708     }
709     BigInteger intVal = new BigInteger(bytes);
710     if(negate) {
711       intVal = intVal.negate();
712     }
713     return new BigDecimal(intVal, scale);
714   }
715 
716   /**
717    * Writes a numeric value.
718    */
719   private void writeNumericValue(ByteBuffer buffer, Object value)
720     throws IOException
721   {
722     Object inValue = value;
723     try {
724       BigDecimal decVal = toBigDecimal(value);
725       inValue = decVal;
726 
727       int signum = decVal.signum();
728       if(signum < 0) {
729         decVal = decVal.negate();
730       }
731 
732       // write sign byte
733       buffer.put((signum < 0) ? NUMERIC_NEGATIVE_BYTE : 0);
734 
735       // adjust scale according to this column type (will cause the an
736       // ArithmeticException if number has too many decimal places)
737       decVal = decVal.setScale(getScale());
738 
739       // check precision
740       if(decVal.precision() > getPrecision()) {
741         throw new IOException(withErrorContext(
742             "Numeric value is too big for specified precision "
743             + getPrecision() + ": " + decVal));
744       }
745     
746       // convert to unscaled BigInteger, big-endian bytes
747       byte[] intValBytes = toUnscaledByteArray(
748           decVal, getType().getFixedSize() - 1);
749       if(buffer.order() != ByteOrder.BIG_ENDIAN) {
750         fixNumericByteOrder(intValBytes);
751       }
752       buffer.put(intValBytes);
753     } catch(ArithmeticException e) {
754       throw (IOException)
755         new IOException(withErrorContext(
756                 "Numeric value '" + inValue + "' out of range"))
757         .initCause(e);
758     }
759   }
760 
761   byte[] toUnscaledByteArray(BigDecimal decVal, int maxByteLen)
762     throws IOException
763   {
764     // convert to unscaled BigInteger, big-endian bytes
765     byte[] intValBytes = decVal.unscaledValue().toByteArray();
766     if(intValBytes.length > maxByteLen) {
767       if((intValBytes[0] == 0) && ((intValBytes.length - 1) == maxByteLen)) {
768         // in order to not return a negative two's complement value,
769         // toByteArray() may return an extra leading 0 byte.  we are working
770         // with unsigned values, so we can drop the extra leading 0
771         intValBytes = ByteUtil.copyOf(intValBytes, 1, maxByteLen);
772       } else {
773         throw new IOException(withErrorContext(
774                                   "Too many bytes for valid BigInteger?"));
775       }
776     } else if(intValBytes.length < maxByteLen) {
777       intValBytes = ByteUtil.copyOf(intValBytes, 0, maxByteLen, 
778                                     (maxByteLen - intValBytes.length));
779     }
780     return intValBytes;
781   }
782 
783   /**
784    * Decodes a date value.
785    */
786   private Date readDateValue(ByteBuffer buffer)
787   {
788     // seems access stores dates in the local timezone.  guess you just hope
789     // you read it in the same timezone in which it was written!
790     long dateBits = buffer.getLong();
791     long time = fromDateDouble(Double.longBitsToDouble(dateBits));
792     return new DateExt(time, dateBits);
793   }
794   
795   /**
796    * Returns a java long time value converted from an access date double.
797    * @usage _advanced_method_
798    */
799   public long fromDateDouble(double value)
800   {
801     long localTime = fromLocalDateDouble(value);
802     return localTime - getFromLocalTimeZoneOffset(localTime);
803   }
804 
805   static long fromLocalDateDouble(double value)
806   {
807     long datePart = ((long)value) * MILLISECONDS_PER_DAY;
808 
809     // the fractional part of the double represents the time.  it is always
810     // a positive fraction of the day (even if the double is negative),
811     // _not_ the time distance from zero (as one would expect with "normal"
812     // numbers).  therefore, we need to do a little number logic to convert
813     // the absolute time fraction into a normal distance from zero number.
814     long timePart = Math.round((Math.abs(value) % 1.0) * 
815                                (double)MILLISECONDS_PER_DAY);
816 
817     long time = datePart + timePart;
818     time -= MILLIS_BETWEEN_EPOCH_AND_1900;
819     return time;
820   }
821 
822   /**
823    * Writes a date value.
824    */
825   private void writeDateValue(ByteBuffer buffer, Object value)
826   {
827     if(value == null) {
828       buffer.putDouble(0d);
829     } else if(value instanceof DateExt) {
830       
831       // this is a Date value previously read from readDateValue().  use the
832       // original bits to store the value so we don't lose any precision
833       buffer.putLong(((DateExt)value).getDateBits());
834       
835     } else {
836       
837       buffer.putDouble(toDateDouble(value));
838     }
839   }
840 
841   /**
842    * Returns an access date double converted from a java Date/Calendar/Number
843    * time value.
844    * @usage _advanced_method_
845    */
846   public double toDateDouble(Object value)
847   {
848     // seems access stores dates in the local timezone.  guess you just
849     // hope you read it in the same timezone in which it was written!
850     long time = toDateLong(value);
851     time += getToLocalTimeZoneOffset(time);
852     return toLocalDateDouble(time);
853   }
854 
855   static double toLocalDateDouble(long time)
856   {
857     time += MILLIS_BETWEEN_EPOCH_AND_1900;
858 
859     if(time < 0L) {
860       // reverse the crazy math described in fromLocalDateDouble
861       long timePart = -time % MILLISECONDS_PER_DAY;
862       if(timePart > 0) {
863         time -= (2 * (MILLISECONDS_PER_DAY - timePart));
864       }
865     }
866 
867     return time / (double)MILLISECONDS_PER_DAY;
868   }
869 
870   /**
871    * @return an appropriate Date long value for the given object
872    */
873   private static long toDateLong(Object value) 
874   {
875     return ((value instanceof Date) ?
876             ((Date)value).getTime() :
877             ((value instanceof Calendar) ?
878              ((Calendar)value).getTimeInMillis() :
879              ((Number)value).longValue()));
880   }
881 
882   /**
883    * Gets the timezone offset from UTC to local time for the given time
884    * (including DST).
885    */
886   private long getToLocalTimeZoneOffset(long time)
887   {
888     Calendar c = getCalendar();
889     c.setTimeInMillis(time);
890     return ((long)c.get(Calendar.ZONE_OFFSET) + c.get(Calendar.DST_OFFSET));
891   }  
892   
893   /**
894    * Gets the timezone offset from local time to UTC for the given time
895    * (including DST).
896    */
897   private long getFromLocalTimeZoneOffset(long time)
898   {
899     // getting from local time back to UTC is a little wonky (and not
900     // guaranteed to get you back to where you started)
901     Calendar c = getCalendar();
902     c.setTimeInMillis(time);
903     // apply the zone offset first to get us closer to the original time
904     c.setTimeInMillis(time - c.get(Calendar.ZONE_OFFSET));
905     return ((long)c.get(Calendar.ZONE_OFFSET) + c.get(Calendar.DST_OFFSET));
906   }  
907   
908   /**
909    * Decodes a GUID value.
910    */
911   private static String readGUIDValue(ByteBuffer buffer, ByteOrder order)
912   {
913     if(order != ByteOrder.BIG_ENDIAN) {
914       byte[] tmpArr = ByteUtil.getBytes(buffer, 16);
915 
916         // the first 3 guid components are integer components which need to
917         // respect endianness, so swap 4-byte int, 2-byte int, 2-byte int
918       ByteUtil.swap4Bytes(tmpArr, 0);
919       ByteUtil.swap2Bytes(tmpArr, 4);
920       ByteUtil.swap2Bytes(tmpArr, 6);
921       buffer = ByteBuffer.wrap(tmpArr);
922     }
923 
924     StringBuilder sb = new StringBuilder(22);
925     sb.append("{");
926     sb.append(ByteUtil.toHexString(buffer, 0, 4,
927                                    false));
928     sb.append("-");
929     sb.append(ByteUtil.toHexString(buffer, 4, 2,
930                                    false));
931     sb.append("-");
932     sb.append(ByteUtil.toHexString(buffer, 6, 2,
933                                    false));
934     sb.append("-");
935     sb.append(ByteUtil.toHexString(buffer, 8, 2,
936                                    false));
937     sb.append("-");
938     sb.append(ByteUtil.toHexString(buffer, 10, 6,
939                                    false));
940     sb.append("}");
941     return (sb.toString());
942   }
943 
944   /**
945    * Writes a GUID value.
946    */
947   private void writeGUIDValue(ByteBuffer buffer, Object value)
948     throws IOException
949   {
950     Matcher m = GUID_PATTERN.matcher(toCharSequence(value));
951     if(!m.matches()) {
952       throw new IOException(withErrorContext("Invalid GUID: " + value));
953     }
954 
955     ByteBuffer origBuffer = null;
956     byte[] tmpBuf = null;
957     if(buffer.order() != ByteOrder.BIG_ENDIAN) {
958       // write to a temp buf so we can do some swapping below
959       origBuffer = buffer;
960       tmpBuf = new byte[16];
961       buffer = ByteBuffer.wrap(tmpBuf);
962     }
963 
964     ByteUtil.writeHexString(buffer, m.group(1));
965     ByteUtil.writeHexString(buffer, m.group(2));
966     ByteUtil.writeHexString(buffer, m.group(3));
967     ByteUtil.writeHexString(buffer, m.group(4));
968     ByteUtil.writeHexString(buffer, m.group(5));
969       
970     if(tmpBuf != null) {
971       // the first 3 guid components are integer components which need to
972       // respect endianness, so swap 4-byte int, 2-byte int, 2-byte int
973       ByteUtil.swap4Bytes(tmpBuf, 0);
974       ByteUtil.swap2Bytes(tmpBuf, 4);
975       ByteUtil.swap2Bytes(tmpBuf, 6);
976       origBuffer.put(tmpBuf);
977     }
978   }
979 
980   /**
981    * Returns {@code true} if the given value is a "guid" value.
982    */
983   static boolean isGUIDValue(Object value) throws IOException {
984     return GUID_PATTERN.matcher(toCharSequence(value)).matches();
985   }
986 
987   /**
988    * Passes the given obj through the currently configured validator for this
989    * column and returns the result.
990    */
991   public Object validate(Object obj) throws IOException {
992     return _validator.validate(this, obj);
993   }
994   
995   /**
996    * Serialize an Object into a raw byte value for this column in little
997    * endian order
998    * @param obj Object to serialize
999    * @return A buffer containing the bytes
1000    * @usage _advanced_method_
1001    */
1002   public ByteBuffer write(Object obj, int remainingRowLength)
1003     throws IOException
1004   {
1005     return write(obj, remainingRowLength, PageChannel.DEFAULT_BYTE_ORDER);
1006   }
1007   
1008   /**
1009    * Serialize an Object into a raw byte value for this column
1010    * @param obj Object to serialize
1011    * @param order Order in which to serialize
1012    * @return A buffer containing the bytes
1013    * @usage _advanced_method_
1014    */
1015   public ByteBuffer write(Object obj, int remainingRowLength, ByteOrder order)
1016     throws IOException
1017   {
1018     if(isRawData(obj)) {
1019       // just slap it right in (not for the faint of heart!)
1020       return ByteBuffer.wrap(((RawData)obj).getBytes());
1021     }
1022 
1023     return writeRealData(obj, remainingRowLength, order);
1024   }
1025 
1026   protected ByteBuffer writeRealData(Object obj, int remainingRowLength, 
1027                                      ByteOrder order)
1028     throws IOException
1029   {
1030     if(!isVariableLength() || !getType().isVariableLength()) {
1031       return writeFixedLengthField(obj, order);
1032     }
1033       
1034     // this is an "inline" var length field
1035     switch(getType()) {
1036     case NUMERIC:
1037       // don't ask me why numerics are "var length" columns...
1038       ByteBuffer buffer = PageChannel.createBuffer(
1039           getType().getFixedSize(), order);
1040       writeNumericValue(buffer, obj);
1041       buffer.flip();
1042       return buffer;
1043 
1044     case TEXT:
1045       return encodeTextValue(
1046           obj, 0, getLengthInUnits(), false).order(order);
1047         
1048     case BINARY:
1049     case UNKNOWN_0D:
1050     case UNSUPPORTED_VARLEN:
1051       // should already be "encoded"
1052       break;
1053     default:
1054       throw new RuntimeException(withErrorContext(
1055               "unexpected inline var length type: " + getType()));
1056     }
1057 
1058     ByteBuffer buffer = ByteBuffer.wrap(toByteArray(obj)).order(order);
1059     return buffer;
1060   }
1061 
1062   /**
1063    * Serialize an Object into a raw byte value for this column
1064    * @param obj Object to serialize
1065    * @param order Order in which to serialize
1066    * @return A buffer containing the bytes
1067    * @usage _advanced_method_
1068    */
1069   protected ByteBuffer writeFixedLengthField(Object obj, ByteOrder order)
1070     throws IOException
1071   {
1072     int size = getType().getFixedSize(_columnLength);
1073 
1074     ByteBuffer buffer = writeFixedLengthField(
1075         obj, PageChannel.createBuffer(size, order));
1076     buffer.flip();
1077     return buffer;
1078   }
1079 
1080   protected ByteBuffer writeFixedLengthField(Object obj, ByteBuffer buffer)
1081     throws IOException
1082   {
1083     // since booleans are not written by this method, it's safe to convert any
1084     // incoming boolean into an integer.
1085     obj = booleanToInteger(obj);
1086 
1087     switch(getType()) {
1088     case BOOLEAN:
1089       //Do nothing
1090       break;
1091     case  BYTE:
1092       buffer.put(toNumber(obj).byteValue());
1093       break;
1094     case INT:
1095       buffer.putShort(toNumber(obj).shortValue());
1096       break;
1097     case LONG:
1098       buffer.putInt(toNumber(obj).intValue());
1099       break;
1100     case MONEY:
1101       writeCurrencyValue(buffer, obj);
1102       break;
1103     case FLOAT:
1104       buffer.putFloat(toNumber(obj).floatValue());
1105       break;
1106     case DOUBLE:
1107       buffer.putDouble(toNumber(obj).doubleValue());
1108       break;
1109     case SHORT_DATE_TIME:
1110       writeDateValue(buffer, obj);
1111       break;
1112     case TEXT:
1113       // apparently text numeric values are also occasionally written as fixed
1114       // length...
1115       int numChars = getLengthInUnits();
1116       // force uncompressed encoding for fixed length text
1117       buffer.put(encodeTextValue(obj, numChars, numChars, true));
1118       break;
1119     case GUID:
1120       writeGUIDValue(buffer, obj);
1121       break;
1122     case NUMERIC:
1123       // yes, that's right, occasionally numeric values are written as fixed
1124       // length...
1125       writeNumericValue(buffer, obj);
1126       break;
1127     case BINARY:
1128     case UNKNOWN_0D:
1129     case UNKNOWN_11:
1130     case COMPLEX_TYPE:
1131       buffer.putInt(toNumber(obj).intValue());
1132       break;
1133     case BIG_INT:
1134       buffer.putLong(toNumber(obj).longValue());
1135       break;
1136     case UNSUPPORTED_FIXEDLEN:
1137       byte[] bytes = toByteArray(obj);
1138       if(bytes.length != getLength()) {
1139         throw new IOException(withErrorContext(
1140                                   "Invalid fixed size binary data, size "
1141                                   + getLength() + ", got " + bytes.length));
1142       }
1143       buffer.put(bytes);
1144       break;
1145     default:
1146       throw new IOException(withErrorContext(
1147                                 "Unsupported data type: " + getType()));
1148     }
1149     return buffer;
1150   }
1151   
1152   /**
1153    * Decodes a compressed or uncompressed text value.
1154    */
1155   String decodeTextValue(byte[] data)
1156     throws IOException
1157   {
1158     // see if data is compressed.  the 0xFF, 0xFE sequence indicates that
1159     // compression is used (sort of, see algorithm below)
1160     boolean isCompressed = ((data.length > 1) &&
1161                             (data[0] == TEXT_COMPRESSION_HEADER[0]) &&
1162                             (data[1] == TEXT_COMPRESSION_HEADER[1]));
1163 
1164     if(isCompressed) {
1165         
1166       // this is a whacky compression combo that switches back and forth
1167       // between compressed/uncompressed using a 0x00 byte (starting in
1168       // compressed mode)
1169       StringBuilder textBuf = new StringBuilder(data.length);
1170       // start after two bytes indicating compression use
1171       int dataStart = TEXT_COMPRESSION_HEADER.length;
1172       int dataEnd = dataStart;
1173       boolean inCompressedMode = true;
1174       while(dataEnd < data.length) {
1175         if(data[dataEnd] == (byte)0x00) {
1176 
1177           // handle current segment
1178           decodeTextSegment(data, dataStart, dataEnd, inCompressedMode,
1179                             textBuf);
1180           inCompressedMode = !inCompressedMode;
1181           ++dataEnd;
1182           dataStart = dataEnd;
1183             
1184         } else {
1185           ++dataEnd;
1186         }
1187       }
1188       // handle last segment
1189       decodeTextSegment(data, dataStart, dataEnd, inCompressedMode, textBuf);
1190 
1191       return textBuf.toString();
1192         
1193     }
1194       
1195     return decodeUncompressedText(data, getCharset());
1196   }
1197 
1198   /**
1199    * Decodes a segnment of a text value into the given buffer according to the
1200    * given status of the segment (compressed/uncompressed).
1201    */
1202   private void decodeTextSegment(byte[] data, int dataStart, int dataEnd,
1203                                  boolean inCompressedMode, 
1204                                  StringBuilder textBuf)
1205   {
1206     if(dataEnd <= dataStart) {
1207       // no data
1208       return;
1209     }
1210     int dataLength = dataEnd - dataStart;
1211 
1212     if(inCompressedMode) {
1213       byte[] tmpData = new byte[dataLength * 2];
1214       int tmpIdx = 0;
1215       for(int i = dataStart; i < dataEnd; ++i) {
1216         tmpData[tmpIdx] = data[i];
1217         tmpIdx += 2;
1218       } 
1219       data = tmpData;
1220       dataStart = 0;
1221       dataLength = data.length;
1222     }
1223 
1224     textBuf.append(decodeUncompressedText(data, dataStart, dataLength,
1225                                           getCharset()));
1226   }
1227 
1228   /**
1229    * @param textBytes bytes of text to decode
1230    * @return the decoded string
1231    */
1232   private static CharBuffer decodeUncompressedText(
1233       byte[] textBytes, int startPos, int length, Charset charset)
1234   {
1235     return charset.decode(ByteBuffer.wrap(textBytes, startPos, length));
1236   }  
1237 
1238   /**
1239    * Encodes a text value, possibly compressing.
1240    */
1241   ByteBuffer encodeTextValue(Object obj, int minChars, int maxChars,
1242                              boolean forceUncompressed)
1243     throws IOException
1244   {
1245     CharSequence text = toCharSequence(obj);
1246     if((text.length() > maxChars) || (text.length() < minChars)) {
1247       throw new IOException(withErrorContext(
1248                             "Text is wrong length for " + getType() +
1249                             " column, max " + maxChars
1250                             + ", min " + minChars + ", got " + text.length()));
1251     }
1252     
1253     // may only compress if column type allows it
1254     if(!forceUncompressed && isCompressedUnicode() &&
1255        (text.length() <= getFormat().MAX_COMPRESSED_UNICODE_SIZE) &&
1256        isUnicodeCompressible(text)) {
1257 
1258       byte[] encodedChars = new byte[TEXT_COMPRESSION_HEADER.length + 
1259                                      text.length()];
1260       encodedChars[0] = TEXT_COMPRESSION_HEADER[0];
1261       encodedChars[1] = TEXT_COMPRESSION_HEADER[1];
1262       for(int i = 0; i < text.length(); ++i) {
1263         encodedChars[i + TEXT_COMPRESSION_HEADER.length] = 
1264           (byte)text.charAt(i);
1265       }
1266       return ByteBuffer.wrap(encodedChars);
1267     }
1268 
1269     return encodeUncompressedText(text, getCharset());
1270   }
1271 
1272   /**
1273    * Returns {@code true} if the given text can be compressed using compressed
1274    * unicode, {@code false} otherwise.
1275    */
1276   private static boolean isUnicodeCompressible(CharSequence text) {
1277     // only attempt to compress > 2 chars (compressing less than 3 chars would
1278     // not result in a space savings due to the 2 byte compression header)
1279     if(text.length() <= TEXT_COMPRESSION_HEADER.length) {
1280       return false;
1281     }
1282     // now, see if it is all compressible characters
1283     for(int i = 0; i < text.length(); ++i) {
1284       char c = text.charAt(i);
1285       if((c < MIN_COMPRESS_CHAR) || (c > MAX_COMPRESS_CHAR)) {
1286         return false;
1287       }
1288     }
1289     return true;
1290   }
1291 
1292   /**
1293    * Constructs a byte containing the flags for this column.
1294    */
1295   private static byte getColumnBitFlags(ColumnBuilder col) {
1296     byte flags = UPDATABLE_FLAG_MASK;
1297     if(!col.isVariableLength()) {
1298       flags |= FIXED_LEN_FLAG_MASK;
1299     }
1300     if(col.isAutoNumber()) {
1301       byte autoNumFlags = 0;
1302       switch(col.getType()) {
1303       case LONG:
1304       case COMPLEX_TYPE:
1305         autoNumFlags = AUTO_NUMBER_FLAG_MASK;
1306         break;
1307       case GUID:
1308         autoNumFlags = AUTO_NUMBER_GUID_FLAG_MASK;
1309         break;
1310       default:
1311         // unknown autonum type
1312       }
1313       flags |= autoNumFlags;
1314     }
1315     if(col.isHyperlink()) {
1316       flags |= HYPERLINK_FLAG_MASK;
1317     }
1318     return flags;
1319   }
1320   
1321   @Override
1322   public String toString() {
1323     ToStringBuilder sb = CustomToStringStyle.builder(this)
1324       .append("name", "(" + _table.getName() + ") " + _name);
1325     byte typeValue = getOriginalDataType();
1326     sb.append("type", "0x" + Integer.toHexString(typeValue) +
1327               " (" + _type + ")")
1328       .append("number", _columnNumber)
1329       .append("length", _columnLength)
1330       .append("variableLength", _variableLength);       
1331     if(_calculated) {
1332       sb.append("calculated", _calculated);
1333     }
1334     if(_type.isTextual()) {
1335       sb.append("compressedUnicode", isCompressedUnicode())
1336         .append("textSortOrder", getTextSortOrder());
1337       if(getTextCodePage() > 0) {
1338         sb.append("textCodePage", getTextCodePage());
1339       }
1340       if(isAppendOnly()) {
1341         sb.append("appendOnly", isAppendOnly());
1342       } 
1343       if(isHyperlink()) {
1344         sb.append("hyperlink", isHyperlink());
1345       } 
1346     }
1347     if(_type.getHasScalePrecision()) {
1348       sb.append("precision", getPrecision())
1349         .append("scale", getScale());
1350     }
1351     if(_autoNumber) {
1352       sb.append("lastAutoNumber", _autoNumberGenerator.getLast());
1353     }
1354     if(getComplexInfo() != null) {
1355       sb.append("complexInfo", getComplexInfo());
1356     }
1357     return sb.toString();
1358   }
1359   
1360   /**
1361    * @param textBytes bytes of text to decode
1362    * @param charset relevant charset
1363    * @return the decoded string
1364    * @usage _advanced_method_
1365    */
1366   public static String decodeUncompressedText(byte[] textBytes, 
1367                                               Charset charset)
1368   {
1369     return decodeUncompressedText(textBytes, 0, textBytes.length, charset)
1370       .toString();
1371   }
1372 
1373   /**
1374    * @param text Text to encode
1375    * @param charset database charset
1376    * @return A buffer with the text encoded
1377    * @usage _advanced_method_
1378    */
1379   public static ByteBuffer encodeUncompressedText(CharSequence text,
1380                                                   Charset charset)
1381   {
1382     CharBuffer cb = ((text instanceof CharBuffer) ? 
1383                      (CharBuffer)text : CharBuffer.wrap(text));
1384     return charset.encode(cb);
1385   }
1386 
1387   
1388   /**
1389    * Orders Columns by column number.
1390    * @usage _general_method_
1391    */
1392   public int compareTo(ColumnImpl other) {
1393     if (_columnNumber > other.getColumnNumber()) {
1394       return 1;
1395     } else if (_columnNumber < other.getColumnNumber()) {
1396       return -1;
1397     } else {
1398       return 0;
1399     }
1400   }
1401   
1402   /**
1403    * @param columns A list of columns in a table definition
1404    * @return The number of variable length columns found in the list
1405    * @usage _advanced_method_
1406    */
1407   public static short countVariableLength(List<ColumnBuilder> columns) {
1408     short rtn = 0;
1409     for (ColumnBuilder col : columns) {
1410       if (col.isVariableLength()) {
1411         rtn++;
1412       }
1413     }
1414     return rtn;
1415   }
1416 
1417   /**
1418    * @return an appropriate BigDecimal representation of the given object.
1419    *         <code>null</code> is returned as 0 and Numbers are converted
1420    *         using their double representation.
1421    */
1422   static BigDecimal toBigDecimal(Object value)
1423   {
1424     if(value == null) {
1425       return BigDecimal.ZERO;
1426     } else if(value instanceof BigDecimal) {
1427       return (BigDecimal)value;
1428     } else if(value instanceof BigInteger) {
1429       return new BigDecimal((BigInteger)value);
1430     } else if(value instanceof Number) {
1431       return new BigDecimal(((Number)value).doubleValue());
1432     }
1433     return new BigDecimal(value.toString());
1434   }
1435 
1436   /**
1437    * @return an appropriate Number representation of the given object.
1438    *         <code>null</code> is returned as 0 and Strings are parsed as
1439    *         Doubles.
1440    */
1441   private static Number toNumber(Object value)
1442   {
1443     if(value == null) {
1444       return BigDecimal.ZERO;
1445     } else if(value instanceof Number) {
1446       return (Number)value;
1447     }
1448     return Double.valueOf(value.toString());
1449   }
1450   
1451   /**
1452    * @return an appropriate CharSequence representation of the given object.
1453    * @usage _advanced_method_
1454    */
1455   public static CharSequence toCharSequence(Object value)
1456     throws IOException
1457   {
1458     if(value == null) {
1459       return null;
1460     } else if(value instanceof CharSequence) {
1461       return (CharSequence)value;
1462     } else if(value instanceof Clob) {
1463       try {
1464         Clob c = (Clob)value;
1465         // note, start pos is 1-based
1466         return c.getSubString(1L, (int)c.length());
1467       } catch(SQLException e) {
1468         throw (IOException)(new IOException(e.getMessage())).initCause(e);
1469       }
1470     } else if(value instanceof Reader) {
1471       char[] buf = new char[8 * 1024];
1472       StringBuilder sout = new StringBuilder();
1473       Reader in = (Reader)value;
1474       int read = 0;
1475       while((read = in.read(buf)) != -1) {
1476         sout.append(buf, 0, read);
1477       }
1478       return sout;
1479     }
1480 
1481     return value.toString();
1482   }
1483 
1484   /**
1485    * @return an appropriate byte[] representation of the given object.
1486    * @usage _advanced_method_
1487    */
1488   public static byte[] toByteArray(Object value)
1489     throws IOException
1490   {
1491     if(value == null) {
1492       return null;
1493     } else if(value instanceof byte[]) {
1494       return (byte[])value;
1495     } else if(value instanceof OleUtil.OleBlobImpl) {
1496       return ((OleUtil.OleBlobImpl)value).getBytes();
1497     } else if(value instanceof Blob) {
1498       try {
1499         Blob b = (Blob)value;
1500         // note, start pos is 1-based
1501         return b.getBytes(1L, (int)b.length());
1502       } catch(SQLException e) {
1503         throw (IOException)(new IOException(e.getMessage())).initCause(e);
1504       }
1505     } else if(value instanceof RawData) {
1506       return ((RawData)value).getBytes();
1507     }
1508 
1509     ByteArrayOutputStream bout = new ByteArrayOutputStream();
1510 
1511     if(value instanceof InputStream) {
1512       ByteUtil.copy((InputStream)value, bout);
1513     } else {
1514       // if all else fails, serialize it
1515       ObjectOutputStream oos = new ObjectOutputStream(bout);
1516       oos.writeObject(value);
1517       oos.close();
1518     }
1519 
1520     return bout.toByteArray();
1521   }
1522 
1523   /**
1524    * Interpret a boolean value (null == false)
1525    * @usage _advanced_method_
1526    */
1527   public static boolean toBooleanValue(Object obj) {
1528     if(obj == null) {
1529       return false;
1530     } else if(obj instanceof Boolean) {
1531       return ((Boolean)obj).booleanValue();
1532     }
1533     return Boolean.parseBoolean(obj.toString());
1534   }
1535   
1536   /**
1537    * Swaps the bytes of the given numeric in place.
1538    */
1539   private static void fixNumericByteOrder(byte[] bytes)
1540   {
1541     // fix endianness of each 4 byte segment
1542     for(int i = 0; i < bytes.length; i+=4) {
1543       ByteUtil.swap4Bytes(bytes, i);
1544     }
1545   }
1546 
1547   /**
1548    * Treat booleans as integers (C-style).
1549    */
1550   protected static Object booleanToInteger(Object obj) {
1551     if (obj instanceof Boolean) {
1552       obj = ((Boolean) obj) ? 1 : 0;
1553     }
1554     return obj;
1555   }
1556 
1557   /**
1558    * Returns a wrapper for raw column data that can be written without
1559    * understanding the data.  Useful for wrapping unparseable data for
1560    * re-writing.
1561    */
1562   public static RawData rawDataWrapper(byte[] bytes) {
1563     return new RawData(bytes);
1564   }
1565 
1566   /**
1567    * Returns {@code true} if the given value is "raw" column data,
1568    * {@code false} otherwise.
1569    * @usage _advanced_method_
1570    */
1571   public static boolean isRawData(Object value) {
1572     return(value instanceof RawData);
1573   }
1574 
1575   /**
1576    * Writes the column definitions into a table definition buffer.
1577    * @param buffer Buffer to write to
1578    */
1579   protected static void writeDefinitions(TableCreator creator, ByteBuffer buffer)
1580     throws IOException
1581   {
1582     // we specifically put the "long variable" values after the normal
1583     // variable length values so that we have a better chance of fitting it
1584     // all (because "long variable" values can go in separate pages)
1585     int longVariableOffset = creator.countNonLongVariableLength();
1586     creator.setColumnOffsets(0, 0, longVariableOffset);
1587 
1588     for (ColumnBuilder col : creator.getColumns()) {
1589       writeDefinition(creator, col, buffer);
1590     }
1591 
1592     for (ColumnBuilder col : creator.getColumns()) {
1593       TableImpl.writeName(buffer, col.getName(), creator.getCharset());
1594     }
1595   }
1596 
1597   protected static void writeDefinition(
1598       TableMutator mutator, ColumnBuilder col, ByteBuffer buffer) 
1599     throws IOException
1600   {
1601     TableMutator.ColumnOffsets colOffsets = mutator.getColumnOffsets();
1602 
1603     buffer.put(col.getType().getValue());
1604     buffer.putInt(TableImpl.MAGIC_TABLE_NUMBER);  //constant magic number
1605     buffer.putShort(col.getColumnNumber());  //Column Number
1606 
1607     if(col.isVariableLength()) {
1608       buffer.putShort(colOffsets.getNextVariableOffset(col));
1609     } else {
1610       buffer.putShort((short) 0);
1611     }
1612 
1613     buffer.putShort(col.getColumnNumber()); //Column Number again
1614 
1615     if(col.getType().isTextual()) {
1616       // this will write 4 bytes (note we don't support writing dbs which
1617       // use the text code page)
1618       writeSortOrder(buffer, col.getTextSortOrder(), mutator.getFormat());
1619     } else {
1620       // note scale/precision not stored for calculated numeric fields
1621       if(col.getType().getHasScalePrecision() && !col.isCalculated()) {
1622         buffer.put(col.getPrecision());  // numeric precision
1623         buffer.put(col.getScale());  // numeric scale
1624       } else {
1625         buffer.put((byte) 0x00); //unused
1626         buffer.put((byte) 0x00); //unused
1627       }
1628       buffer.putShort((short) 0); //Unknown
1629     }
1630 
1631     buffer.put(getColumnBitFlags(col)); // misc col flags
1632 
1633     // note access doesn't seem to allow unicode compression for calced fields
1634     if(col.isCalculated()) {
1635       buffer.put(CALCULATED_EXT_FLAG_MASK);
1636     } else if (col.isCompressedUnicode()) {  //Compressed
1637       buffer.put(COMPRESSED_UNICODE_EXT_FLAG_MASK);
1638     } else {
1639       buffer.put((byte)0);
1640     }
1641 
1642     buffer.putInt(0); //Unknown, but always 0.
1643 
1644     //Offset for fixed length columns
1645     if(col.isVariableLength()) {
1646       buffer.putShort((short) 0);
1647     } else {
1648       buffer.putShort(colOffsets.getNextFixedOffset(col));
1649     }
1650 
1651     if(!col.getType().isLongValue()) {
1652       short length = col.getLength();
1653       if(col.isCalculated()) {
1654         // calced columns have additional value overhead
1655         if(!col.getType().isVariableLength() || 
1656            col.getType().getHasScalePrecision()) {
1657           length = CalculatedColumnUtil.CALC_FIXED_FIELD_LEN; 
1658         } else {
1659           length += CalculatedColumnUtil.CALC_EXTRA_DATA_LEN;
1660         }
1661       }
1662       buffer.putShort(length); //Column length
1663     } else {
1664       buffer.putShort((short)0x0000); // unused
1665     }
1666   }
1667 
1668   protected static void writeColUsageMapDefinitions(
1669       TableCreator creator, ByteBuffer buffer)
1670     throws IOException
1671   {
1672     // write long value column usage map references
1673     for(ColumnBuilder lvalCol : creator.getLongValueColumns()) {
1674       writeColUsageMapDefinition(creator, lvalCol, buffer);
1675     }
1676   }
1677 
1678   protected static void writeColUsageMapDefinition(
1679       TableMutator creator, ColumnBuilder lvalCol, ByteBuffer buffer)
1680     throws IOException
1681   {
1682     TableMutator.ColumnState colState = creator.getColumnState(lvalCol);
1683 
1684     buffer.putShort(lvalCol.getColumnNumber());
1685       
1686     // owned pages umap (both are on same page)
1687     buffer.put(colState.getUmapOwnedRowNumber());
1688     ByteUtil.put3ByteInt(buffer, colState.getUmapPageNumber());
1689     // free space pages umap
1690     buffer.put(colState.getUmapFreeRowNumber());
1691     ByteUtil.put3ByteInt(buffer, colState.getUmapPageNumber());
1692   }
1693 
1694   /**
1695    * Reads the sort order info from the given buffer from the given position.
1696    */
1697   static SortOrder readSortOrder(ByteBuffer buffer, int position,
1698                                  JetFormat format)
1699   {
1700     short value = buffer.getShort(position);
1701     byte version = 0;
1702     if(format.SIZE_SORT_ORDER == 4) {
1703       version = buffer.get(position + 3);
1704     }
1705 
1706     if(value == 0) {
1707       // probably a file we wrote, before handling sort order
1708       return format.DEFAULT_SORT_ORDER;
1709     }
1710     
1711     if(value == GENERAL_SORT_ORDER_VALUE) {
1712       if(version == GENERAL_LEGACY_SORT_ORDER.getVersion()) {
1713         return GENERAL_LEGACY_SORT_ORDER;
1714       }
1715       if(version == GENERAL_SORT_ORDER.getVersion()) {
1716         return GENERAL_SORT_ORDER;
1717       }
1718     }
1719     return new SortOrder(value, version);
1720   }
1721 
1722   /**
1723    * Reads the column cade page info from the given buffer, if supported for
1724    * this db.
1725    */
1726   static short readCodePage(ByteBuffer buffer, int offset, JetFormat format)
1727   {
1728       int cpOffset = format.OFFSET_COLUMN_CODE_PAGE;
1729       return ((cpOffset >= 0) ? buffer.getShort(offset + cpOffset) : 0);
1730   }
1731 
1732   /**
1733    * Read the extra flags field for a column definition.
1734    */
1735   static byte readExtraFlags(ByteBuffer buffer, int offset, JetFormat format)
1736   {
1737     int extFlagsOffset = format.OFFSET_COLUMN_EXT_FLAGS;
1738     return ((extFlagsOffset >= 0) ? buffer.get(offset + extFlagsOffset) : 0);
1739   }
1740   
1741   /**
1742    * Writes the sort order info to the given buffer at the current position.
1743    */
1744   private static void writeSortOrder(ByteBuffer buffer, SortOrder sortOrder,
1745                                      JetFormat format) {
1746     if(sortOrder == null) {
1747       sortOrder = format.DEFAULT_SORT_ORDER;
1748     }
1749     buffer.putShort(sortOrder.getValue());      
1750     if(format.SIZE_SORT_ORDER == 4) {
1751       buffer.put((byte)0x00); // unknown
1752       buffer.put(sortOrder.getVersion());
1753     }
1754   }
1755 
1756   /**
1757    * Returns {@code true} if the value is immutable, {@code false} otherwise.
1758    * This only handles values that are returned from the {@link #read} method.
1759    */
1760   static boolean isImmutableValue(Object value) {
1761     // for now, the only mutable value this class returns is byte[]
1762     return !(value instanceof byte[]);
1763   }
1764 
1765   /**
1766    * Converts the given value to the "internal" representation for the given
1767    * data type.
1768    */
1769   public static Object toInternalValue(DataType dataType, Object value)
1770     throws IOException
1771   {
1772     if(value == null) {
1773       return null;
1774     }
1775 
1776     switch(dataType) {
1777     case BOOLEAN:
1778       return ((value instanceof Boolean) ? value : toBooleanValue(value));
1779     case BYTE:
1780       return ((value instanceof Byte) ? value : toNumber(value).byteValue());
1781     case INT:
1782       return ((value instanceof Short) ? value : 
1783               toNumber(value).shortValue());
1784     case LONG:
1785       return ((value instanceof Integer) ? value : 
1786               toNumber(value).intValue());
1787     case MONEY:
1788       return toBigDecimal(value);
1789     case FLOAT:
1790       return ((value instanceof Float) ? value : 
1791               toNumber(value).floatValue());
1792     case DOUBLE:
1793       return ((value instanceof Double) ? value : 
1794               toNumber(value).doubleValue());
1795     case SHORT_DATE_TIME:
1796       return ((value instanceof DateExt) ? value :
1797               new Date(toDateLong(value)));
1798     case TEXT:
1799     case MEMO:
1800     case GUID:
1801       return ((value instanceof String) ? value : 
1802               toCharSequence(value).toString());
1803     case NUMERIC:
1804       return toBigDecimal(value);
1805     case COMPLEX_TYPE:
1806       // leave alone for now?
1807       return value;
1808     case BIG_INT:
1809       return ((value instanceof Long) ? value : 
1810               toNumber(value).longValue());
1811     default:
1812       // some variation of binary data
1813       return toByteArray(value);
1814     }
1815   }
1816 
1817   String withErrorContext(String msg) {
1818     return withErrorContext(msg, getDatabase(), getTable().getName(), getName());
1819   }
1820 
1821   private static String withErrorContext(
1822       String msg, DatabaseImpl db, String tableName, String colName) {
1823     return msg + " (Db=" + db.getName() + ";Table=" + tableName + ";Column=" + 
1824       colName + ")";
1825   }
1826 
1827   /**
1828    * Date subclass which stashes the original date bits, in case we attempt to
1829    * re-write the value (will not lose precision).
1830    */
1831   private static final class DateExt extends Date
1832   {
1833     private static final long serialVersionUID = 0L;
1834 
1835     /** cached bits of the original date value */
1836     private transient final long _dateBits;
1837 
1838     private DateExt(long time, long dateBits) {
1839       super(time);
1840       _dateBits = dateBits;
1841     }
1842 
1843     public long getDateBits() {
1844       return _dateBits;
1845     }
1846     
1847     private Object writeReplace() throws ObjectStreamException {
1848       // if we are going to serialize this Date, convert it back to a normal
1849       // Date (in case it is restored outside of the context of jackcess)
1850       return new Date(super.getTime());
1851     }
1852   }
1853 
1854   /**
1855    * Wrapper for raw column data which can be re-written.
1856    */
1857   private static class RawData implements Serializable
1858   {
1859     private static final long serialVersionUID = 0L;
1860 
1861     private final byte[] _bytes;
1862 
1863     private RawData(byte[] bytes) {
1864       _bytes = bytes;
1865     }
1866 
1867     private byte[] getBytes() {
1868       return _bytes;
1869     }
1870 
1871     @Override
1872     public String toString() {
1873       return CustomToStringStyle.valueBuilder(this)
1874         .append(null, getBytes())
1875         .toString();
1876     }
1877 
1878     private Object writeReplace() throws ObjectStreamException {
1879       // if we are going to serialize this, convert it back to a normal
1880       // byte[] (in case it is restored outside of the context of jackcess)
1881       return getBytes();
1882     }
1883   }
1884 
1885   /**
1886    * Base class for the supported autonumber types.
1887    * @usage _advanced_class_
1888    */
1889   public abstract class AutoNumberGenerator
1890   {
1891     protected AutoNumberGenerator() {}
1892 
1893     /**
1894      * Returns the last autonumber generated by this generator.  Only valid
1895      * after a call to {@link Table#addRow}, otherwise undefined.
1896      */
1897     public abstract Object getLast();
1898 
1899     /**
1900      * Returns the next autonumber for this generator.
1901      * <p>
1902      * <i>Warning, calling this externally will result in this value being
1903      * "lost" for the table.</i>
1904      */
1905     public abstract Object getNext(TableImpl.WriteRowState writeRowState);
1906 
1907     /**
1908      * Returns a valid autonumber for this generator.
1909      * <p>
1910      * <i>Warning, calling this externally may result in this value being
1911      * "lost" for the table.</i>
1912      */
1913     public abstract Object handleInsert(
1914         TableImpl.WriteRowState writeRowState, Object inRowValue) 
1915       throws IOException;
1916 
1917     /**
1918      * Restores a previous autonumber generated by this generator.
1919      */
1920     public abstract void restoreLast(Object last);
1921     
1922     /**
1923      * Returns the type of values generated by this generator.
1924      */
1925     public abstract DataType getType();
1926   }
1927 
1928   private final class LongAutoNumberGenerator extends AutoNumberGenerator
1929   {
1930     private LongAutoNumberGenerator() {}
1931 
1932     @Override
1933     public Object getLast() {
1934       // the table stores the last long autonumber used
1935       return getTable().getLastLongAutoNumber();
1936     }
1937 
1938     @Override
1939     public Object getNext(TableImpl.WriteRowState writeRowState) {
1940       // the table stores the last long autonumber used
1941       return getTable().getNextLongAutoNumber();
1942     }
1943 
1944     @Override
1945     public Object handleInsert(TableImpl.WriteRowState writeRowState,
1946                                Object inRowValue) 
1947       throws IOException
1948     {
1949       int inAutoNum = toNumber(inRowValue).intValue();
1950       if(inAutoNum <= INVALID_AUTO_NUMBER && !getTable().isAllowAutoNumberInsert()) {
1951         throw new IOException(withErrorContext(
1952                 "Invalid auto number value " + inAutoNum));
1953       }
1954       // the table stores the last long autonumber used
1955       getTable().adjustLongAutoNumber(inAutoNum);
1956       return inAutoNum;
1957     }
1958 
1959     @Override
1960     public void restoreLast(Object last) {
1961       if(last instanceof Integer) {
1962         getTable().restoreLastLongAutoNumber((Integer)last);
1963       }
1964     }
1965     
1966     @Override
1967     public DataType getType() {
1968       return DataType.LONG;
1969     }
1970   }
1971 
1972   private final class GuidAutoNumberGenerator extends AutoNumberGenerator
1973   {
1974     private Object _lastAutoNumber;
1975 
1976     private GuidAutoNumberGenerator() {}
1977 
1978     @Override
1979     public Object getLast() {
1980       return _lastAutoNumber;
1981     }
1982 
1983     @Override
1984     public Object getNext(TableImpl.WriteRowState writeRowState) {
1985       // format guids consistently w/ Column.readGUIDValue()
1986       _lastAutoNumber = "{" + UUID.randomUUID() + "}";
1987       return _lastAutoNumber;
1988     }
1989 
1990     @Override
1991     public Object handleInsert(TableImpl.WriteRowState writeRowState,
1992                                Object inRowValue) 
1993       throws IOException
1994     {
1995       _lastAutoNumber = toCharSequence(inRowValue);
1996       return _lastAutoNumber;
1997     }
1998 
1999     @Override
2000     public void restoreLast(Object last) {
2001       _lastAutoNumber = null;
2002     }
2003     
2004     @Override
2005     public DataType getType() {
2006       return DataType.GUID;
2007     }
2008   }
2009 
2010   private final class ComplexTypeAutoNumberGenerator extends AutoNumberGenerator
2011   {
2012     private ComplexTypeAutoNumberGenerator() {}
2013 
2014     @Override
2015     public Object getLast() {
2016       // the table stores the last ComplexType autonumber used
2017       return getTable().getLastComplexTypeAutoNumber();
2018     }
2019 
2020     @Override
2021     public Object getNext(TableImpl.WriteRowState writeRowState) {
2022       // same value is shared across all ComplexType values in a row
2023       int nextComplexAutoNum = writeRowState.getComplexAutoNumber();
2024       if(nextComplexAutoNum <= INVALID_AUTO_NUMBER) {
2025         // the table stores the last ComplexType autonumber used
2026         nextComplexAutoNum = getTable().getNextComplexTypeAutoNumber();
2027         writeRowState.setComplexAutoNumber(nextComplexAutoNum);
2028       }
2029       return new ComplexValueForeignKeyImpl(ColumnImpl.this, 
2030                                             nextComplexAutoNum);
2031     }
2032 
2033     @Override
2034     public Object handleInsert(TableImpl.WriteRowState writeRowState,
2035                                Object inRowValue) 
2036       throws IOException
2037     {
2038       ComplexValueForeignKey inComplexFK = null;
2039       if(inRowValue instanceof ComplexValueForeignKey) {
2040         inComplexFK = (ComplexValueForeignKey)inRowValue;
2041       } else {
2042         inComplexFK = new ComplexValueForeignKeyImpl(
2043             ColumnImpl.this, toNumber(inRowValue).intValue());
2044       }
2045 
2046       if(inComplexFK.getColumn() != ColumnImpl.this) {
2047         throw new IOException(withErrorContext(
2048                 "Wrong column for complex value foreign key, found " +
2049                 inComplexFK.getColumn().getName()));
2050       }
2051       if(inComplexFK.get() < 1) {
2052         throw new IOException(withErrorContext(
2053                 "Invalid complex value foreign key value " + inComplexFK.get()));
2054       }
2055       // same value is shared across all ComplexType values in a row
2056       int prevRowValue = writeRowState.getComplexAutoNumber();
2057       if(prevRowValue <= INVALID_AUTO_NUMBER) {
2058         writeRowState.setComplexAutoNumber(inComplexFK.get());
2059       } else if(prevRowValue != inComplexFK.get()) {
2060         throw new IOException(withErrorContext(
2061                 "Inconsistent complex value foreign key values: found " +
2062                 prevRowValue + ", given " + inComplexFK));
2063       }
2064 
2065       // the table stores the last ComplexType autonumber used
2066       getTable().adjustComplexTypeAutoNumber(inComplexFK.get());
2067 
2068       return inComplexFK;
2069     }
2070 
2071     @Override
2072     public void restoreLast(Object last) {
2073       if(last instanceof ComplexValueForeignKey) {
2074         getTable().restoreLastComplexTypeAutoNumber(
2075             ((ComplexValueForeignKey)last).get());
2076       }
2077     }
2078     
2079     @Override
2080     public DataType getType() {
2081       return DataType.COMPLEX_TYPE;
2082     }
2083   }
2084   
2085   private final class UnsupportedAutoNumberGenerator extends AutoNumberGenerator
2086   {
2087     private final DataType _genType;
2088     
2089     private UnsupportedAutoNumberGenerator(DataType genType) {
2090       _genType = genType;
2091     }
2092     
2093     @Override
2094     public Object getLast() {
2095       return null;
2096     }
2097 
2098     @Override
2099     public Object getNext(TableImpl.WriteRowState writeRowState) {
2100       throw new UnsupportedOperationException();
2101     }
2102 
2103     @Override
2104     public Object handleInsert(TableImpl.WriteRowState writeRowState,
2105                                Object inRowValue) {
2106       throw new UnsupportedOperationException();
2107     }
2108 
2109     @Override
2110     public void restoreLast(Object last) {
2111       throw new UnsupportedOperationException();
2112     }
2113     
2114     @Override
2115     public DataType getType() {
2116       return _genType;
2117     }
2118   }
2119 
2120   
2121   /**
2122    * Information about the sort order (collation) for a textual column.
2123    * @usage _intermediate_class_
2124    */
2125   public static final class SortOrder
2126   {
2127     private final short _value;
2128     private final byte _version;
2129     
2130     public SortOrder(short value, byte version) {
2131       _value = value;
2132       _version = version;
2133     }
2134 
2135     public short getValue() {
2136       return _value;
2137     }
2138 
2139     public byte getVersion() {
2140       return _version;
2141     }
2142 
2143     @Override
2144     public int hashCode() {
2145       return _value;
2146     }
2147 
2148     @Override
2149     public boolean equals(Object o) {
2150       return ((this == o) ||
2151               ((o != null) && (getClass() == o.getClass()) &&
2152                (_value == ((SortOrder)o)._value) &&
2153                (_version == ((SortOrder)o)._version)));
2154     }
2155 
2156     @Override
2157     public String toString() {
2158       return CustomToStringStyle.valueBuilder(this)
2159         .append(null, _value + "(" + _version + ")")
2160         .toString();
2161     }
2162   }
2163 
2164   /**
2165    * Utility struct for passing params through ColumnImpl constructors.
2166    */
2167   static final class InitArgs
2168   {
2169     public final TableImpl table;
2170     public final ByteBuffer buffer;
2171     public final int offset;
2172     public final String name;
2173     public final int displayIndex;
2174     public final byte colType;
2175     public final byte flags;
2176     public final byte extFlags;
2177     public DataType type;
2178 
2179     InitArgs(TableImpl table, ByteBuffer buffer, int offset, String name,
2180              int displayIndex) {
2181       this.table = table;
2182       this.buffer = buffer;
2183       this.offset = offset;
2184       this.name = name;
2185       this.displayIndex = displayIndex;
2186       
2187       this.colType = buffer.get(offset + table.getFormat().OFFSET_COLUMN_TYPE);
2188       this.flags = buffer.get(offset + table.getFormat().OFFSET_COLUMN_FLAGS);
2189       this.extFlags = readExtraFlags(buffer, offset, table.getFormat());
2190     }
2191   }
2192 }