/*

 Copyright (c) 2005 Health Market Science, Inc.

 This library is free software; you can redistribute it and/or

 modify it under the terms of the GNU Lesser General Public

 License as published by the Free Software Foundation; either

 version 2.1 of the License, or (at your option) any later version.

 This library is distributed in the hope that it will be useful,

 but WITHOUT ANY WARRANTY; without even the implied warranty of

 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

 Lesser General Public License for more details.

 You should have received a copy of the GNU Lesser General Public

 License along with this library; if not, write to the Free Software

 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307

 USA

 You can contact Health Market Science at info@healthmarketscience.com

 or at the following address:

 Health Market Science

 2700 Horizon Drive

 Suite 200

 King of Prussia, PA 19406

 */

 package com.healthmarketscience.jackcess;

 import java.io.IOException;

 import java.nio.ByteBuffer;

 import java.util.ArrayList;

 import java.util.Arrays;

 import java.util.Collection;

 import java.util.Collections;

 import java.util.Comparator;

 import java.util.Iterator;

 import java.util.LinkedHashMap;

 import java.util.List;

 import java.util.Map;

 import org.apache.commons.logging.Log;

 import org.apache.commons.logging.LogFactory;

 /**

  * A single database table

  * <p>

  * Is not thread-safe.

  * 

  * @author Tim McCune

  */

 public class Table

   implements Iterable<Map<String, Object>>

 {

   private static final Log LOG = LogFactory.getLog(Table.class);

   private static final short OFFSET_MASK = (short)0x1FFF;

   private static final short DELETED_ROW_MASK = (short)0x8000;

   private static final short OVERFLOW_ROW_MASK = (short)0x4000;

   /** Table type code for system tables */

   public static final byte TYPE_SYSTEM = 0x53;

   /** Table type code for user tables */

   public static final byte TYPE_USER = 0x4e;

   /** comparator which sorts variable length columns vased on their index into

       the variable length offset table */

   private static final Comparator<Column> VAR_LEN_COLUMN_COMPARATOR =

     new Comparator<Column>() {

       public int compare(Column c1, Column c2) {

         return ((c1.getVarLenTableIndex() < c2.getVarLenTableIndex()) ? -1 :

                 ((c1.getVarLenTableIndex() > c2.getVarLenTableIndex()) ? 1 :

                  0));

       }

     };

   /** owning database */

   private final Database _database;

   /** Type of the table (either TYPE_SYSTEM or TYPE_USER) */

   private byte _tableType;

   /** Number of indexes on the table */

   private int _indexCount;

   /** Number of index slots for the table */

   private int _indexSlotCount;

   /** Number of rows in the table */

   private int _rowCount;

   /** last auto number for the table */

   private int _lastAutoNumber;

   /** page number of the definition of this table */

   private final int _tableDefPageNumber;

   /** max Number of columns in the table (includes previous deletions) */

   private short _maxColumnCount;

   /** max Number of variable columns in the table */

   private short _maxVarColumnCount;

   /** List of columns in this table, ordered by column number */

   private List<Column> _columns = new ArrayList<Column>();

   /** List of variable length columns in this table, ordered by offset */

   private List<Column> _varColumns = new ArrayList<Column>();

   /** List of indexes on this table */

   private List<Index> _indexes = new ArrayList<Index>();

   /** Table name as stored in Database */

   private final String _name;

   /** Usage map of pages that this table owns */

   private UsageMap _ownedPages;

   /** Usage map of pages that this table owns with free space on them */

   private UsageMap _freeSpacePages;

   /** modification count for the table, keeps row-states up-to-date */

   private int _modCount;

   /** page buffer used to update data pages when adding rows */

   private final TempPageHolder _addRowBufferH =

     TempPageHolder.newHolder(TempBufferHolder.Type.SOFT);

   /** page buffer used to update the table def page */

   private final TempPageHolder _tableDefBufferH =

     TempPageHolder.newHolder(TempBufferHolder.Type.SOFT);

   /** buffer used to writing single rows of data */

   private final TempBufferHolder _singleRowBufferH =

     TempBufferHolder.newHolder(TempBufferHolder.Type.SOFT, true);

   /** "buffer" used to writing multi rows of data (will create new buffer on

       every call) */

   private final TempBufferHolder _multiRowBufferH =

     TempBufferHolder.newHolder(TempBufferHolder.Type.NONE, true);

   /** page buffer used to write out-of-line "long value" data */

   private final TempPageHolder _longValueBufferH =

     TempPageHolder.newHolder(TempBufferHolder.Type.SOFT);

   /** for now, "big index support" is optional */

   private final boolean _useBigIndex;

   /** common cursor for iterating through the table, kept here for historic

       reasons */

   private Cursor _cursor;

   /**

    * Only used by unit tests

    */

   Table(boolean testing, List<Column> columns) throws IOException {

     if(!testing) {

       throw new IllegalArgumentException();

     }

     _database = null;

     _tableDefPageNumber = PageChannel.INVALID_PAGE_NUMBER;

     _name = null;

     _useBigIndex = false;

     setColumns(columns);

   }

   /**

    * @param database database which owns this table

    * @param tableBuffer Buffer to read the table with

    * @param pageNumber Page number of the table definition

    * @param name Table name

    * @param useBigIndex whether or not "big index support" should be enabled

    *                    for the table

    */

   protected Table(Database database, ByteBuffer tableBuffer,

                   int pageNumber, String name, boolean useBigIndex)

   throws IOException

   {

     _database = database;

     _tableDefPageNumber = pageNumber;

     _name = name;

     _useBigIndex = useBigIndex; 

     int nextPage = tableBuffer.getInt(getFormat().OFFSET_NEXT_TABLE_DEF_PAGE);

     ByteBuffer nextPageBuffer = null;

     while (nextPage != 0) {

       if (nextPageBuffer == null) {

         nextPageBuffer = getPageChannel().createPageBuffer();

       }

       getPageChannel().readPage(nextPageBuffer, nextPage);

       nextPage = nextPageBuffer.getInt(getFormat().OFFSET_NEXT_TABLE_DEF_PAGE);

       ByteBuffer newBuffer = getPageChannel().createBuffer(

           tableBuffer.capacity() + getFormat().PAGE_SIZE - 8);

       newBuffer.put(tableBuffer);

       newBuffer.put(nextPageBuffer.array(), 8, getFormat().PAGE_SIZE - 8);

       tableBuffer = newBuffer;

       tableBuffer.flip();

     }

     readTableDefinition(tableBuffer);

     tableBuffer = null;

     // setup common cursor

     _cursor = Cursor.createCursor(this);

   }

   /**

    * @return The name of the table

    */

   public String getName() {

     return _name;

   }

   public boolean doUseBigIndex() {

     return _useBigIndex;

   }

   public int getMaxColumnCount() {

     return _maxColumnCount;

   }

   public int getColumnCount() {

     return _columns.size();

   }

   public Database getDatabase() {

     return _database;

   }

   public JetFormat getFormat() {

     return getDatabase().getFormat();

   }

   public PageChannel getPageChannel() {

     return getDatabase().getPageChannel();

   }

   protected int getTableDefPageNumber() {

     return _tableDefPageNumber;

   }

   public RowState createRowState() {

     return new RowState(TempBufferHolder.Type.HARD);

   }

   protected UsageMap.PageCursor getOwnedPagesCursor() {

     return _ownedPages.cursor();

   }

   protected TempPageHolder getLongValueBuffer() {

     return _longValueBufferH;

   }

   /**

    * @return All of the columns in this table (unmodifiable List)

    */

   public List<Column> getColumns() {

     return Collections.unmodifiableList(_columns);

   }

   /**

    * @return the column with the given name

    */

   public Column getColumn(String name) {

     for(Column column : _columns) {

       if(column.getName().equals(name)) {

         return column;

       }

     }

     throw new IllegalArgumentException("Column with name " + name +

                                        " does not exist in this table");

   }

   /**

    * Only called by unit tests

    */

   private void setColumns(List<Column> columns) {

     _columns = columns;

     int colIdx = 0;

     int varLenIdx = 0;

     int fixedOffset = 0;

     for(Column col : _columns) {

       col.setColumnNumber((short)colIdx);

       col.setColumnIndex(colIdx++);

       if(col.isVariableLength()) {

         col.setVarLenTableIndex(varLenIdx++);

         _varColumns.add(col);

       } else {

         col.setFixedDataOffset(fixedOffset);

         fixedOffset += col.getType().getFixedSize();

       }

     }

     _maxColumnCount = (short)_columns.size();

     _maxVarColumnCount = (short)_varColumns.size();

   }

   /**

    * @return All of the Indexes on this table (unmodifiable List)

    */

   public List<Index> getIndexes() {

     return Collections.unmodifiableList(_indexes);

   }

   /**

    * @return the index with the given name

    */

   public Index getIndex(String name) {

     for(Index index : _indexes) {

       if(index.getName().equals(name)) {

         return index;

       }

     }

     throw new IllegalArgumentException("Index with name " + name +

                                        " does not exist on this table");

   }

   /**

    * Only called by unit tests

    */

   int getIndexSlotCount() {

     return _indexSlotCount;

   }

   /**

    * After calling this method, getNextRow will return the first row in the

    * table

    */

   public void reset() {

     _cursor.reset();

   }

   /**

    * Delete the current row (retrieved by a call to {@link #getNextRow()}).

    */

   public void deleteCurrentRow() throws IOException {

     _cursor.deleteCurrentRow();

   }

   /**

    * Delete the row on which the given rowState is currently positioned.

    */

   public void deleteRow(RowState rowState, RowId rowId) throws IOException {

     requireValidRowId(rowId);

     // ensure that the relevant row state is up-to-date

     ByteBuffer rowBuffer = positionAtRowHeader(rowState, rowId);

     requireNonDeletedRow(rowState, rowId);

     // delete flag always gets set in the "header" row (even if data is on

     // overflow row)

     int pageNumber = rowState.getHeaderRowId().getPageNumber();

     int rowNumber = rowState.getHeaderRowId().getRowNumber();

     // use any read rowValues to help update the indexes

     Object[] rowValues = (!_indexes.isEmpty() ?

                           rowState.getRowValues() : null);

     int rowIndex = getRowStartOffset(rowNumber, getFormat());

     rowBuffer.putShort(rowIndex, (short)(rowBuffer.getShort(rowIndex)

                                       | DELETED_ROW_MASK | OVERFLOW_ROW_MASK));

     writeDataPage(rowBuffer, pageNumber);

     // update the indexes

     for(Index index : _indexes) {

       index.deleteRow(rowValues, rowId);

     }

     // make sure table def gets updated

     updateTableDefinition(-1);

   }

   /**

    * @return The next row in this table (Column name -> Column value)

    */

   public Map<String, Object> getNextRow() throws IOException {

     return getNextRow(null);

   }

   /**

    * @param columnNames Only column names in this collection will be returned

    * @return The next row in this table (Column name -> Column value)

    */

   public Map<String, Object> getNextRow(Collection<String> columnNames) 

     throws IOException

   {

     return _cursor.getNextRow(columnNames);

   }

   /**

    * Reads a single column from the given row.

    */

   public Object getRowValue(RowState rowState, RowId rowId, Column column)

     throws IOException

   {

     if(this != column.getTable()) {

       throw new IllegalArgumentException(

           "Given column " + column + " is not from this table");

     }

     requireValidRowId(rowId);

     // position at correct row

     ByteBuffer rowBuffer = positionAtRowData(rowState, rowId);

     requireNonDeletedRow(rowState, rowId);

     Object value = getRowColumn(rowBuffer, getRowNullMask(rowBuffer), column);

     // cache the row values in order to be able to update the index on row

     // deletion.  note, most of the returned values are immutable, except

     // for binary data (returned as byte[]), but binary data shouldn't be

     // indexed anyway.

     rowState.setRowValue(column.getColumnIndex(), value);

     return value;

   }

   /**

    * Reads some columns from the given row.

    * @param columnNames Only column names in this collection will be returned

    */

   public Map<String, Object> getRow(

       RowState rowState, RowId rowId, Collection<String> columnNames)

     throws IOException

   {

     requireValidRowId(rowId);

     // position at correct row

     ByteBuffer rowBuffer = positionAtRowData(rowState, rowId);

     requireNonDeletedRow(rowState, rowId);

     return getRow(rowState, rowBuffer, getRowNullMask(rowBuffer), _columns,

                   columnNames);

   }

   /**

    * Reads the row data from the given row buffer.  Leaves limit unchanged.

    * Saves parsed row values to the given rowState.

    */

   private static Map<String, Object> getRow(

       RowState rowState,

       ByteBuffer rowBuffer,

       NullMask nullMask,

       Collection<Column> columns,

       Collection<String> columnNames)

     throws IOException

   {

     Map<String, Object> rtn = new LinkedHashMap<String, Object>(

         columns.size());

     for(Column column : columns) {

       if((columnNames == null) || (columnNames.contains(column.getName()))) {

         // Add the value to the row data

         Object value = getRowColumn(rowBuffer, nullMask, column);

         rtn.put(column.getName(), value);

         // cache the row values in order to be able to update the index on row

         // deletion.  note, most of the returned values are immutable, except

         // for binary data (returned as byte[]), but binary data shouldn't be

         // indexed anyway.

         rowState.setRowValue(column.getColumnIndex(), value);

       }

     }

     return rtn;

   }

   /**

    * Reads the column data from the given row buffer.  Leaves limit unchanged.

    */

   private static Object getRowColumn(ByteBuffer rowBuffer,

                                      NullMask nullMask,

                                      Column column)

     throws IOException

   {

     boolean isNull = nullMask.isNull(column);

     if(column.getType() == DataType.BOOLEAN) {

       return Boolean.valueOf(!isNull);  //Boolean values are stored in the null mask

     } else if(isNull) {

       // well, that's easy!

       return null;

     }

     // reset position to row start

     rowBuffer.reset();

     // locate the column data bytes

     int rowStart = rowBuffer.position();

     int colDataPos = 0;

     int colDataLen = 0;

     if(!column.isVariableLength()) {

       // read fixed length value (non-boolean at this point)

       int dataStart = rowStart + 2;

       colDataPos = dataStart + column.getFixedDataOffset();

       colDataLen = column.getType().getFixedSize();

     } else {

       // read var length value

       int varColumnOffsetPos =

         (rowBuffer.limit() - nullMask.byteSize() - 4) -

         (column.getVarLenTableIndex() * 2);

       short varDataStart = rowBuffer.getShort(varColumnOffsetPos);

       short varDataEnd = rowBuffer.getShort(varColumnOffsetPos - 2);

       colDataPos = rowStart + varDataStart;

       colDataLen = varDataEnd - varDataStart;

     }

     // grab the column data

     byte[] columnData = new byte[colDataLen];

     rowBuffer.position(colDataPos);

     rowBuffer.get(columnData);

     // parse the column data

     return column.read(columnData);

   }

   /**

    * Reads the null mask from the given row buffer.  Leaves limit unchanged.

    */

   private static NullMask getRowNullMask(ByteBuffer rowBuffer)

     throws IOException

   {

     // reset position to row start

     rowBuffer.reset();

     short columnCount = rowBuffer.getShort(); // Number of columns in this row

     // read null mask

     NullMask nullMask = new NullMask(columnCount);

     rowBuffer.position(rowBuffer.limit() - nullMask.byteSize());  //Null mask at end

     nullMask.read(rowBuffer);

     return nullMask;

   }

   /**

    * Sets a new buffer to the correct row header page using the given rowState

    * according to the given rowId.  Deleted state is

    * determined, but overflow row pointers are not followed.

    * 

    * @return a ByteBuffer of the relevant page, or null if row was invalid

    */

   public static ByteBuffer positionAtRowHeader(RowState rowState,

                                                RowId rowId)

     throws IOException

   {

     ByteBuffer rowBuffer = rowState.setHeaderRow(rowId);

     if(rowState.isAtHeaderRow()) {

       // this task has already been accomplished

       return rowBuffer;

     }

     if(!rowState.isValid()) {

       // this was an invalid page/row

       rowState.setStatus(RowStateStatus.AT_HEADER);

       return null;

     }

     // note, we don't use findRowStart here cause we need the unmasked value

     short rowStart = rowBuffer.getShort(

         getRowStartOffset(rowId.getRowNumber(),

                           rowState.getTable().getFormat()));

     // check the deleted, overflow flags for the row (the "real" flags are

     // always set on the header row)

     RowStatus rowStatus = RowStatus.NORMAL;

     if(isDeletedRow(rowStart)) {

       rowStatus = RowStatus.DELETED;

     } else if(isOverflowRow(rowStart)) {

       rowStatus = RowStatus.OVERFLOW;

     }

     rowState.setRowStatus(rowStatus);

     rowState.setStatus(RowStateStatus.AT_HEADER);

     return rowBuffer;

   }

   /**

    * Sets the position and limit in a new buffer using the given rowState

    * according to the given row number and row end, following overflow row

    * pointers as necessary.

    * 

    * @return a ByteBuffer narrowed to the actual row data, or null if row was

    *         invalid or deleted

    */

   public static ByteBuffer positionAtRowData(RowState rowState,

                                              RowId rowId)

     throws IOException

   {

     positionAtRowHeader(rowState, rowId);

     if(!rowState.isValid() || rowState.isDeleted()) {

       // row is invalid or deleted

       rowState.setStatus(RowStateStatus.AT_FINAL);

       return null;

     }

     ByteBuffer rowBuffer = rowState.getFinalPage();

     int rowNum = rowState.getFinalRowId().getRowNumber();

     JetFormat format = rowState.getTable().getFormat();

     if(rowState.isAtFinalRow()) {

       // we've already found the final row data

       return PageChannel.narrowBuffer(

           rowBuffer,

           findRowStart(rowBuffer, rowNum, format),

           findRowEnd(rowBuffer, rowNum, format));

     }

     while(true) {

       // note, we don't use findRowStart here cause we need the unmasked value

       short rowStart = rowBuffer.getShort(getRowStartOffset(rowNum, format));

       short rowEnd = findRowEnd(rowBuffer, rowNum, format);

       // note, at this point we know the row is not deleted, so ignore any

       // subsequent deleted flags (as overflow rows are always marked deleted

       // anyway)

       boolean overflowRow = isOverflowRow(rowStart);

       // now, strip flags from rowStart offset

       rowStart = (short)(rowStart & OFFSET_MASK);

       if (overflowRow) {

         if((rowEnd - rowStart) < 4) {

           throw new IOException("invalid overflow row info");

         }

         // Overflow page.  the "row" data in the current page points to

         // another page/row

         int overflowRowNum = ByteUtil.getUnsignedByte(rowBuffer, rowStart);

         int overflowPageNum = ByteUtil.get3ByteInt(rowBuffer, rowStart + 1);

         rowBuffer = rowState.setOverflowRow(

             new RowId(overflowPageNum, overflowRowNum));

         rowNum = overflowRowNum;

       } else {

         rowState.setStatus(RowStateStatus.AT_FINAL);

         return PageChannel.narrowBuffer(rowBuffer, rowStart, rowEnd);

       }

     }

   }

   /**

    * Calls <code>reset</code> on this table and returns an unmodifiable

    * Iterator which will iterate through all the rows of this table.  Use of

    * the Iterator follows the same restrictions as a call to

    * <code>getNextRow</code>.

    * @throws IllegalStateException if an IOException is thrown by one of the

    *         operations, the actual exception will be contained within

    */

   public Iterator<Map<String, Object>> iterator()

   {

     return iterator(null);

   }

   /**

    * Calls <code>reset</code> on this table and returns an unmodifiable

    * Iterator which will iterate through all the rows of this table, returning

    * only the given columns.  Use of the Iterator follows the same

    * restrictions as a call to <code>getNextRow</code>.

    * @throws IllegalStateException if an IOException is thrown by one of the

    *         operations, the actual exception will be contained within