1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28 package com.healthmarketscience.jackcess;
29
30 import java.io.IOException;
31 import java.nio.ByteBuffer;
32 import java.util.BitSet;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36
37
38
39
40
41 public class UsageMap
42 {
43 private static final Log LOG = LogFactory.getLog(UsageMap.class);
44
45
46 public static final byte MAP_TYPE_INLINE = 0x0;
47
48 public static final byte MAP_TYPE_REFERENCE = 0x1;
49
50
51 private static final int INVALID_BIT_INDEX = -1;
52
53
54 private final Database _database;
55
56 private final int _tablePageNum;
57
58 private int _startOffset;
59
60 private final short _rowStart;
61
62 private int _startPage;
63
64 private int _endPage;
65
66 private BitSet _pageNumbers = new BitSet();
67
68 private final ByteBuffer _tableBuffer;
69
70
71 private int _modCount;
72
73
74 private Handler _handler;
75
76
77
78
79
80
81
82 private UsageMap(Database database, ByteBuffer tableBuffer,
83 int pageNum, short rowStart)
84 throws IOException
85 {
86 _database = database;
87 _tableBuffer = tableBuffer;
88 _tablePageNum = pageNum;
89 _rowStart = rowStart;
90 _tableBuffer.position(_rowStart + getFormat().OFFSET_USAGE_MAP_START);
91 _startOffset = _tableBuffer.position();
92 if (LOG.isDebugEnabled()) {
93 LOG.debug("Usage map block:\n" + ByteUtil.toHexString(_tableBuffer, _rowStart,
94 tableBuffer.limit() - _rowStart));
95 }
96 }
97
98 public Database getDatabase() {
99 return _database;
100 }
101
102 public JetFormat getFormat() {
103 return getDatabase().getFormat();
104 }
105
106 public PageChannel getPageChannel() {
107 return getDatabase().getPageChannel();
108 }
109
110
111
112
113
114
115
116
117 public static UsageMap read(Database database, int pageNum,
118 int rowNum, boolean assumeOutOfRangeBitsOn)
119 throws IOException
120 {
121 JetFormat format = database.getFormat();
122 PageChannel pageChannel = database.getPageChannel();
123 ByteBuffer tableBuffer = pageChannel.createPageBuffer();
124 pageChannel.readPage(tableBuffer, pageNum);
125 short rowStart = Table.findRowStart(tableBuffer, rowNum, format);
126 int rowEnd = Table.findRowEnd(tableBuffer, rowNum, format);
127 tableBuffer.limit(rowEnd);
128 byte mapType = tableBuffer.get(rowStart);
129 UsageMap rtn = new UsageMap(database, tableBuffer, pageNum, rowStart);
130 rtn.initHandler(mapType, assumeOutOfRangeBitsOn);
131 return rtn;
132 }
133
134 private void initHandler(byte mapType, boolean assumeOutOfRangeBitsOn)
135 throws IOException
136 {
137 if (mapType == MAP_TYPE_INLINE) {
138 _handler = new InlineHandler(assumeOutOfRangeBitsOn);
139 } else if (mapType == MAP_TYPE_REFERENCE) {
140 _handler = new ReferenceHandler();
141 } else {
142 throw new IOException("Unrecognized map type: " + mapType);
143 }
144 }
145
146 public PageCursor cursor() {
147 return new PageCursor();
148 }
149
150 protected short getRowStart() {
151 return _rowStart;
152 }
153
154 protected int getRowEnd() {
155 return getTableBuffer().limit();
156 }
157
158 protected void setStartOffset(int startOffset) {
159 _startOffset = startOffset;
160 }
161
162 protected int getStartOffset() {
163 return _startOffset;
164 }
165
166 protected ByteBuffer getTableBuffer() {
167 return _tableBuffer;
168 }
169
170 protected int getTablePageNumber() {
171 return _tablePageNum;
172 }
173
174 protected int getStartPage() {
175 return _startPage;
176 }
177
178 protected int getEndPage() {
179 return _endPage;
180 }
181
182 protected BitSet getPageNumbers() {
183 return _pageNumbers;
184 }
185
186 protected void setPageRange(int newStartPage, int newEndPage) {
187 _startPage = newStartPage;
188 _endPage = newEndPage;
189 }
190
191 protected boolean isPageWithinRange(int pageNumber)
192 {
193 return((pageNumber >= _startPage) && (pageNumber < _endPage));
194 }
195
196 protected int getFirstPageNumber() {
197 return bitIndexToPageNumber(getNextBitIndex(-1), RowId.LAST_PAGE_NUMBER);
198 }
199
200 protected int getNextPageNumber(int curPage) {
201 return bitIndexToPageNumber(
202 getNextBitIndex(pageNumberToBitIndex(curPage)),
203 RowId.LAST_PAGE_NUMBER);
204 }
205
206 protected int getNextBitIndex(int curIndex) {
207 return _pageNumbers.nextSetBit(curIndex + 1);
208 }
209
210 protected int getLastPageNumber() {
211 return bitIndexToPageNumber(getPrevBitIndex(_pageNumbers.length()),
212 RowId.FIRST_PAGE_NUMBER);
213 }
214
215 protected int getPrevPageNumber(int curPage) {
216 return bitIndexToPageNumber(
217 getPrevBitIndex(pageNumberToBitIndex(curPage)),
218 RowId.FIRST_PAGE_NUMBER);
219 }
220
221 protected int getPrevBitIndex(int curIndex) {
222 --curIndex;
223 while((curIndex >= 0) && !_pageNumbers.get(curIndex)) {
224 --curIndex;
225 }
226 return curIndex;
227 }
228
229 protected int bitIndexToPageNumber(int bitIndex,
230 int invalidPageNumber) {
231 return((bitIndex >= 0) ? (_startPage + bitIndex) : invalidPageNumber);
232 }
233
234 protected int pageNumberToBitIndex(int pageNumber) {
235 return((pageNumber >= 0) ? (pageNumber - _startPage) :
236 INVALID_BIT_INDEX);
237 }
238
239 protected void clearTableAndPages()
240 {
241
242 _pageNumbers.clear();
243 _startPage = 0;
244 _endPage = 0;
245 ++_modCount;
246
247
248 int tableStart = getRowStart() + 1;
249 int tableEnd = getRowEnd();
250 ByteUtil.clearRange(_tableBuffer, tableStart, tableEnd);
251 }
252
253 protected void writeTable()
254 throws IOException
255 {
256
257 getPageChannel().writePage(_tableBuffer, _tablePageNum, _rowStart);
258 }
259
260
261
262
263 protected void processMap(ByteBuffer buffer, int bufferStartPage)
264 {
265 int byteCount = 0;
266 while (buffer.hasRemaining()) {
267 byte b = buffer.get();
268 if(b != (byte)0) {
269 for (int i = 0; i < 8; i++) {
270 if ((b & (1 << i)) != 0) {
271 int pageNumberOffset = (byteCount * 8 + i) + bufferStartPage;
272 int pageNumber = bitIndexToPageNumber(
273 pageNumberOffset,
274 PageChannel.INVALID_PAGE_NUMBER);
275 if(!isPageWithinRange(pageNumber)) {
276 throw new IllegalStateException(
277 "found page number " + pageNumber
278 + " in usage map outside of expected range " +
279 _startPage + " to " + _endPage);
280 }
281 _pageNumbers.set(pageNumberOffset);
282 }
283 }
284 }
285 byteCount++;
286 }
287 }
288
289
290
291
292 public boolean containsPageNumber(int pageNumber) {
293 return _handler.containsPageNumber(pageNumber);
294 }
295
296
297
298
299 public void addPageNumber(int pageNumber) throws IOException {
300 ++_modCount;
301 _handler.addOrRemovePageNumber(pageNumber, true);
302 }
303
304
305
306
307 public void removePageNumber(int pageNumber) throws IOException {
308 ++_modCount;
309 _handler.addOrRemovePageNumber(pageNumber, false);
310 }
311
312 protected void updateMap(int absolutePageNumber,
313 int bufferRelativePageNumber,
314 ByteBuffer buffer, boolean add)
315 throws IOException
316 {
317
318 int offset = bufferRelativePageNumber / 8;
319 int bitmask = 1 << (bufferRelativePageNumber % 8);
320 byte b = buffer.get(_startOffset + offset);
321
322
323 int pageNumberOffset = pageNumberToBitIndex(absolutePageNumber);
324 boolean isOn = _pageNumbers.get(pageNumberOffset);
325 if(isOn == add) {
326 throw new IOException("Page number " + absolutePageNumber + " already " +
327 ((add) ? "added to" : "removed from") +
328 " usage map, expected range " +
329 _startPage + " to " + _endPage);
330 }
331
332
333 if (add) {
334 b |= bitmask;
335 _pageNumbers.set(pageNumberOffset);
336 } else {
337 b &= ~bitmask;
338 _pageNumbers.clear(pageNumberOffset);
339 }
340 buffer.put(_startOffset + offset, b);
341 }
342
343
344
345
346 private void promoteInlineHandlerToReferenceHandler(int newPageNumber)
347 throws IOException
348 {
349
350 int oldStartPage = _startPage;
351 BitSet oldPageNumbers = (BitSet)_pageNumbers.clone();
352
353
354 clearTableAndPages();
355
356
357 _tableBuffer.put(getRowStart(), MAP_TYPE_REFERENCE);
358
359
360 writeTable();
361
362
363 _handler = new ReferenceHandler();
364
365
366 reAddPages(oldStartPage, oldPageNumbers, newPageNumber);
367 }
368
369 private void reAddPages(int oldStartPage, BitSet oldPageNumbers,
370 int newPageNumber)
371 throws IOException
372 {
373
374 for(int i = oldPageNumbers.nextSetBit(0); i >= 0;
375 i = oldPageNumbers.nextSetBit(i + 1)) {
376 addPageNumber(oldStartPage + i);
377 }
378
379 if(newPageNumber > PageChannel.INVALID_PAGE_NUMBER) {
380
381 addPageNumber(newPageNumber);
382 }
383 }
384
385 @Override
386 public String toString() {
387 StringBuilder builder = new StringBuilder(
388 "page numbers (range " + _startPage + " " + _endPage + "): [");
389 PageCursor pCursor = cursor();
390 while(true) {
391 int nextPage = pCursor.getNextPage();
392 if(nextPage < 0) {
393 break;
394 }
395 builder.append(nextPage).append(", ");
396 }
397 builder.append("]");
398 return builder.toString();
399 }
400
401 private abstract class Handler
402 {
403 protected Handler() {
404 }
405
406 public boolean containsPageNumber(int pageNumber) {
407 return(isPageWithinRange(pageNumber) &&
408 getPageNumbers().get(pageNumberToBitIndex(pageNumber)));
409 }
410
411
412
413
414
415 public abstract void addOrRemovePageNumber(int pageNumber, boolean add)
416 throws IOException;
417 }
418
419
420
421
422
423
424
425
426
427 private class InlineHandler extends Handler
428 {
429 private final boolean _assumeOutOfRangeBitsOn;
430 private final int _maxInlinePages;
431
432 private InlineHandler(boolean assumeOutOfRangeBitsOn)
433 throws IOException
434 {
435 _assumeOutOfRangeBitsOn = assumeOutOfRangeBitsOn;
436 _maxInlinePages = (getInlineDataEnd() - getInlineDataStart()) * 8;
437 int startPage = getTableBuffer().getInt(getRowStart() + 1);
438 setInlinePageRange(startPage);
439 processMap(getTableBuffer(), 0);
440 }
441
442 private int getMaxInlinePages() {
443 return _maxInlinePages;
444 }
445
446 private int getInlineDataStart() {
447 return getRowStart() + getFormat().OFFSET_USAGE_MAP_START;
448 }
449
450 private int getInlineDataEnd() {
451 return getRowEnd();
452 }
453
454
455
456
457
458 private void setInlinePageRange(int startPage) {
459 setPageRange(startPage, startPage + getMaxInlinePages());
460 }
461
462 @Override
463 public boolean containsPageNumber(int pageNumber) {
464 return(super.containsPageNumber(pageNumber) ||
465 (_assumeOutOfRangeBitsOn && (pageNumber >= 0) &&
466 !isPageWithinRange(pageNumber)));
467 }
468
469 @Override
470 public void addOrRemovePageNumber(int pageNumber, boolean add)
471 throws IOException
472 {
473 if(isPageWithinRange(pageNumber)) {
474
475
476 int bufferRelativePageNumber = pageNumberToBitIndex(pageNumber);
477 updateMap(pageNumber, bufferRelativePageNumber, getTableBuffer(), add);
478
479 writeTable();
480
481 } else {
482
483
484
485 int firstPage = getFirstPageNumber();
486 int lastPage = getLastPageNumber();
487
488 if(add) {
489
490
491
492
493
494 if(!_assumeOutOfRangeBitsOn) {
495
496
497 if(firstPage <= PageChannel.INVALID_PAGE_NUMBER) {
498
499 firstPage = pageNumber;
500 lastPage = pageNumber;
501 } else if(pageNumber > lastPage) {
502 lastPage = pageNumber;
503 } else {
504 firstPage = pageNumber;
505 }
506 if((lastPage - firstPage + 1) < getMaxInlinePages()) {
507
508
509 moveToNewStartPage(firstPage, pageNumber);
510
511 } else {
512
513
514 promoteInlineHandlerToReferenceHandler(pageNumber);
515 }
516 }
517 } else {
518
519
520 if(_assumeOutOfRangeBitsOn) {
521
522
523
524
525
526 if((firstPage <= PageChannel.INVALID_PAGE_NUMBER) ||
527 (pageNumber > lastPage)) {
528
529
530 moveToNewStartPageForRemove(firstPage, pageNumber);
531
532 }
533
534 } else {
535
536
537
538 throw new IOException("Page number " + pageNumber +
539 " already removed from usage map" +
540 ", expected range " +
541 _startPage + " to " + _endPage);
542 }
543 }
544
545 }
546 }
547
548
549
550
551
552
553
554 private void moveToNewStartPage(int newStartPage, int newPageNumber)
555 throws IOException
556 {
557 int oldStartPage = getStartPage();
558 BitSet oldPageNumbers = (BitSet)getPageNumbers().clone();
559
560
561 clearTableAndPages();
562
563
564 ByteBuffer tableBuffer = getTableBuffer();
565 tableBuffer.position(getRowStart() + 1);
566 tableBuffer.putInt(newStartPage);
567
568
569 writeTable();
570
571
572 setInlinePageRange(newStartPage);
573
574
575 reAddPages(oldStartPage, oldPageNumbers, newPageNumber);
576 }
577
578
579
580
581
582
583
584
585
586 private void moveToNewStartPageForRemove(int firstPage, int newPageNumber)
587 throws IOException
588 {
589 int oldEndPage = getEndPage();
590 int newStartPage =
591 ((firstPage <= PageChannel.INVALID_PAGE_NUMBER) ? newPageNumber :
592
593 (newPageNumber - (getMaxInlinePages() / 2)));
594
595
596 moveToNewStartPage(newStartPage, PageChannel.INVALID_PAGE_NUMBER);
597
598 if(firstPage <= PageChannel.INVALID_PAGE_NUMBER) {
599
600
601 ByteUtil.fillRange(_tableBuffer, getInlineDataStart(),
602 getInlineDataEnd());
603
604
605 writeTable();
606
607
608 getPageNumbers().set(0, getMaxInlinePages());
609
610 } else {
611
612
613 for(int i = oldEndPage; i < getEndPage(); ++i) {
614 addPageNumber(i);
615 }
616 }
617
618
619 removePageNumber(newPageNumber);
620 }
621 }
622
623
624
625
626
627
628
629
630 private class ReferenceHandler extends Handler
631 {
632
633 private final TempPageHolder _mapPageHolder =
634 TempPageHolder.newHolder(TempBufferHolder.Type.SOFT);
635
636 private ReferenceHandler()
637 throws IOException
638 {
639 int numUsagePages = (getRowEnd() - getRowStart() - 1) / 4;
640 setStartOffset(getFormat().OFFSET_USAGE_MAP_PAGE_DATA);
641 setPageRange(0, (numUsagePages * getMaxPagesPerUsagePage()));
642
643
644
645
646 for (int i = 0; i < numUsagePages; i++) {
647 int mapPageNum = getTableBuffer().getInt(
648 calculateMapPagePointerOffset(i));
649 if (mapPageNum > 0) {
650 ByteBuffer mapPageBuffer =
651 _mapPageHolder.setPage(getPageChannel(), mapPageNum);
652 byte pageType = mapPageBuffer.get();
653 if (pageType != PageTypes.USAGE_MAP) {
654 throw new IOException("Looking for usage map at page " +
655 mapPageNum + ", but page type is " +
656 pageType);
657 }
658 mapPageBuffer.position(getFormat().OFFSET_USAGE_MAP_PAGE_DATA);
659 processMap(mapPageBuffer, (getMaxPagesPerUsagePage() * i));
660 }
661 }
662 }
663
664 private int getMaxPagesPerUsagePage() {
665 return((getFormat().PAGE_SIZE - getFormat().OFFSET_USAGE_MAP_PAGE_DATA)
666 * 8);
667 }
668
669 @Override
670 public void addOrRemovePageNumber(int pageNumber, boolean add)
671 throws IOException
672 {
673 if(!isPageWithinRange(pageNumber)) {
674 throw new IOException("Page number " + pageNumber +
675 " is out of supported range");
676 }
677 int pageIndex = (pageNumber / getMaxPagesPerUsagePage());
678 int mapPageNum = getTableBuffer().getInt(
679 calculateMapPagePointerOffset(pageIndex));
680 ByteBuffer mapPageBuffer = null;
681 if(mapPageNum > 0) {
682 mapPageBuffer = _mapPageHolder.setPage(getPageChannel(), mapPageNum);
683 } else {
684
685 mapPageBuffer = createNewUsageMapPage(pageIndex);
686 mapPageNum = _mapPageHolder.getPageNumber();
687 }
688 updateMap(pageNumber,
689 (pageNumber - (getMaxPagesPerUsagePage() * pageIndex)),
690 mapPageBuffer, add);
691 getPageChannel().writePage(mapPageBuffer, mapPageNum);
692 }
693
694
695
696
697
698
699 private ByteBuffer createNewUsageMapPage(int pageIndex) throws IOException
700 {
701 ByteBuffer mapPageBuffer = _mapPageHolder.setNewPage(getPageChannel());
702 mapPageBuffer.put(PageTypes.USAGE_MAP);
703 mapPageBuffer.put((byte) 0x01);
704 mapPageBuffer.putShort((short) 0);
705 int mapPageNum = _mapPageHolder.getPageNumber();
706 getTableBuffer().putInt(calculateMapPagePointerOffset(pageIndex),
707 mapPageNum);
708 writeTable();
709 return mapPageBuffer;
710 }
711
712 private int calculateMapPagePointerOffset(int pageIndex) {
713 return getRowStart() + getFormat().OFFSET_REFERENCE_MAP_PAGE_NUMBERS +
714 (pageIndex * 4);
715 }
716 }
717
718
719
720
721
722 public final class PageCursor
723 {
724
725 private final DirHandler _forwardDirHandler = new ForwardDirHandler();
726
727 private final DirHandler _reverseDirHandler = new ReverseDirHandler();
728
729 private int _curPageNumber;
730
731 private int _prevPageNumber;
732
733
734
735 private int _lastModCount;
736
737 private PageCursor() {
738 reset();
739 }
740
741 public UsageMap getUsageMap() {
742 return UsageMap.this;
743 }
744
745
746
747
748 private DirHandler getDirHandler(boolean moveForward) {
749 return (moveForward ? _forwardDirHandler : _reverseDirHandler);
750 }
751
752
753
754
755
756 public boolean isUpToDate() {
757 return(UsageMap.this._modCount == _lastModCount);
758 }
759
760
761
762
763
764 public int getNextPage() {
765 return getAnotherPage(Cursor.MOVE_FORWARD);
766 }
767
768
769
770
771
772 public int getPreviousPage() {
773 return getAnotherPage(Cursor.MOVE_REVERSE);
774 }
775
776
777
778
779 private int getAnotherPage(boolean moveForward) {
780 DirHandler handler = getDirHandler(moveForward);
781 if(_curPageNumber == handler.getEndPageNumber()) {
782 if(!isUpToDate()) {
783 restorePosition(_prevPageNumber);
784
785 } else {
786
787 return _curPageNumber;
788 }
789 }
790
791 checkForModification();
792
793 _prevPageNumber = _curPageNumber;
794 _curPageNumber = handler.getAnotherPageNumber(_curPageNumber);
795 return _curPageNumber;
796 }
797
798
799
800
801
802 public void reset() {
803 beforeFirst();
804 }
805
806
807
808
809
810 public void beforeFirst() {
811 reset(Cursor.MOVE_FORWARD);
812 }
813
814
815
816
817
818 public void afterLast() {
819 reset(Cursor.MOVE_REVERSE);
820 }
821
822
823
824
825 protected void reset(boolean moveForward) {
826 _curPageNumber = getDirHandler(moveForward).getBeginningPageNumber();
827 _prevPageNumber = _curPageNumber;
828 _lastModCount = UsageMap.this._modCount;
829 }
830
831
832
833
834
835 private void restorePosition(int curPageNumber)
836 {
837 restorePosition(curPageNumber, _curPageNumber);
838 }
839
840
841
842
843 protected void restorePosition(int curPageNumber, int prevPageNumber)
844 {
845 if((curPageNumber != _curPageNumber) ||
846 (prevPageNumber != _prevPageNumber))
847 {
848 _prevPageNumber = updatePosition(prevPageNumber);
849 _curPageNumber = updatePosition(curPageNumber);
850 _lastModCount = UsageMap.this._modCount;
851 } else {
852 checkForModification();
853 }
854 }
855
856
857
858
859 private void checkForModification() {
860 if(!isUpToDate()) {
861 _prevPageNumber = updatePosition(_prevPageNumber);
862 _curPageNumber = updatePosition(_curPageNumber);
863 _lastModCount = UsageMap.this._modCount;
864 }
865 }
866
867 private int updatePosition(int pageNumber) {
868 if(pageNumber < UsageMap.this.getFirstPageNumber()) {
869 pageNumber = RowId.FIRST_PAGE_NUMBER;
870 } else if(pageNumber > UsageMap.this.getLastPageNumber()) {
871 pageNumber = RowId.LAST_PAGE_NUMBER;
872 }
873 return pageNumber;
874 }
875
876 @Override
877 public String toString() {
878 return getClass().getSimpleName() + " CurPosition " + _curPageNumber +
879 ", PrevPosition " + _prevPageNumber;
880 }
881
882
883
884
885
886
887 private abstract class DirHandler {
888 public abstract int getAnotherPageNumber(int curPageNumber);
889 public abstract int getBeginningPageNumber();
890 public abstract int getEndPageNumber();
891 }
892
893
894
895
896 private final class ForwardDirHandler extends DirHandler {
897 @Override
898 public int getAnotherPageNumber(int curPageNumber) {
899 if(curPageNumber == getBeginningPageNumber()) {
900 return UsageMap.this.getFirstPageNumber();
901 }
902 return UsageMap.this.getNextPageNumber(curPageNumber);
903 }
904 @Override
905 public int getBeginningPageNumber() {
906 return RowId.FIRST_PAGE_NUMBER;
907 }
908 @Override
909 public int getEndPageNumber() {
910 return RowId.LAST_PAGE_NUMBER;
911 }
912 }
913
914
915
916
917 private final class ReverseDirHandler extends DirHandler {
918 @Override
919 public int getAnotherPageNumber(int curPageNumber) {
920 if(curPageNumber == getBeginningPageNumber()) {
921 return UsageMap.this.getLastPageNumber();
922 }
923 return UsageMap.this.getPrevPageNumber(curPageNumber);
924 }
925 @Override
926 public int getBeginningPageNumber() {
927 return RowId.LAST_PAGE_NUMBER;
928 }
929 @Override
930 public int getEndPageNumber() {
931 return RowId.FIRST_PAGE_NUMBER;
932 }
933 }
934
935 }
936
937 }