View Javadoc
1   /*
2   Copyright (c) 2008 Health Market Science, Inc.
3   
4   Licensed under the Apache License, Version 2.0 (the "License");
5   you may not use this file except in compliance with the License.
6   You may obtain a copy of the License at
7   
8       http://www.apache.org/licenses/LICENSE-2.0
9   
10  Unless required by applicable law or agreed to in writing, software
11  distributed under the License is distributed on an "AS IS" BASIS,
12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  See the License for the specific language governing permissions and
14  limitations under the License.
15  */
16  
17  package com.healthmarketscience.jackcess.impl;
18  
19  import java.io.File;
20  import java.lang.reflect.Field;
21  import java.nio.ByteBuffer;
22  import java.util.Arrays;
23  import java.util.HashMap;
24  import java.util.Map;
25  import java.util.TreeMap;
26  import java.util.regex.Matcher;
27  import java.util.regex.Pattern;
28  
29  import com.healthmarketscience.jackcess.ColumnBuilder;
30  import com.healthmarketscience.jackcess.Cursor;
31  import com.healthmarketscience.jackcess.CursorBuilder;
32  import com.healthmarketscience.jackcess.DataType;
33  import com.healthmarketscience.jackcess.Database;
34  import com.healthmarketscience.jackcess.DateTimeType;
35  import com.healthmarketscience.jackcess.Index;
36  import com.healthmarketscience.jackcess.Row;
37  import com.healthmarketscience.jackcess.Table;
38  import com.healthmarketscience.jackcess.TableBuilder;
39  import static com.healthmarketscience.jackcess.impl.JetFormatTest.*;
40  import junit.framework.TestCase;
41  import static com.healthmarketscience.jackcess.TestUtil.*;
42  
43  /**
44   * @author James Ahlborn
45   */
46  public class IndexCodesTest extends TestCase {
47  
48    private static final Map<Character,String> SPECIAL_CHARS =
49      new HashMap<Character,String>();
50    static {
51      SPECIAL_CHARS.put('\b', "\\b");
52      SPECIAL_CHARS.put('\t', "\\t");
53      SPECIAL_CHARS.put('\n', "\\n");
54      SPECIAL_CHARS.put('\f', "\\f");
55      SPECIAL_CHARS.put('\r', "\\r");
56      SPECIAL_CHARS.put('\"', "\\\"");
57      SPECIAL_CHARS.put('\'', "\\'");
58      SPECIAL_CHARS.put('\\', "\\\\");
59    }
60  
61    public IndexCodesTest(String name) throws Exception {
62      super(name);
63    }
64  
65    public void testIndexCodes() throws Exception
66    {
67      for (final TestDB testDB : TestDB.getSupportedForBasename(Basename.INDEX_CODES, true)) {
68        Database db = openMem(testDB);
69        db.setDateTimeType(DateTimeType.DATE);
70  
71        for(Table t : db) {
72          for(Index index : t.getIndexes()) {
73    //         System.out.println("Checking " + t.getName() + "." + index.getName());
74            checkIndexEntries(testDB, t, index);
75          }
76        }
77  
78        db.close();
79      }
80    }
81  
82    public static void checkIndexEntries(final TestDB testDB, Table t, Index index) throws Exception
83    {
84  //         index.initialize();
85  //         System.out.println("Ind " + index);
86  
87      Cursor cursor = CursorBuilder.createCursor(index);
88      while(cursor.moveToNextRow()) {
89  
90        Row row = cursor.getCurrentRow();
91  
92        Object data = row.get("data");
93        if((testDB.getExpectedFileFormat() == Database.FileFormat.V1997) &&
94           (data instanceof String) && ((String)data).contains("\uFFFD")) {
95          // this row has a character not supported in the v1997 charset
96          continue;
97        }
98  
99        Cursor.Position curPos = cursor.getSavepoint().getCurrentPosition();
100       boolean success = false;
101       try {
102         findRow(testDB, t, index, row, curPos);
103         success = true;
104       } finally {
105         if(!success) {
106           System.out.println("CurPos: " + curPos);
107           System.out.println("Value: " + row + ": " +
108                              toUnicodeStr(row.get("data")));
109         }
110       }
111     }
112 
113   }
114 
115   private static void findRow(final TestDB testDB, Table t, Index index,
116                               Row expectedRow,
117                               Cursor.Position expectedPos)
118     throws Exception
119   {
120     Object[] idxRow = ((IndexImpl)index).constructIndexRow(expectedRow);
121     Cursor cursor = CursorBuilder.createCursor(index, idxRow, idxRow);
122 
123     Cursor.Position startPos = cursor.getSavepoint().getCurrentPosition();
124 
125     cursor.beforeFirst();
126     while(cursor.moveToNextRow()) {
127       Row row = cursor.getCurrentRow();
128       if(expectedRow.equals(row)) {
129         // verify that the entries are indeed equal
130         Cursor.Position curPos = cursor.getSavepoint().getCurrentPosition();
131         assertEquals(entryToString(expectedPos), entryToString(curPos));
132         return;
133       }
134     }
135 
136     // TODO long rows not handled completely yet in V2010
137     // seems to truncate entry at 508 bytes with some trailing 2 byte seq
138     if((testDB != null) &&
139        (testDB.getExpectedFileFormat() == Database.FileFormat.V2010)) {
140       String rowId = expectedRow.getString("name");
141       String tName = t.getName();
142       if(("Table11".equals(tName) || "Table11_desc".equals(tName)) &&
143          ("row10".equals(rowId) || "row11".equals(rowId) ||
144           "row12".equals(rowId))) {
145         System.out.println(
146             "TODO long rows not handled completely yet in V2010: " + tName +
147                            ", " + rowId);
148         return;
149       }
150     }
151 
152     fail("testDB: " + testDB + ";\nCould not find expected row " + expectedRow + " starting at " +
153          entryToString(startPos));
154   }
155 
156 
157   //////
158   //
159   // The code below is for use in reverse engineering index entries.
160   //
161   //////
162 
163   public void testNothing() throws Exception {
164     // keep this so build doesn't fail if other tests are disabled
165   }
166 
167   public void x_testCreateIsoFile() throws Exception
168   {
169     Database db = create(Database.FileFormat.V2000, true);
170 
171     Table t = new TableBuilder("test")
172       .addColumn(new ColumnBuilder("row", DataType.TEXT))
173       .addColumn(new ColumnBuilder("data", DataType.TEXT))
174       .toTable(db);
175 
176     for(int i = 0; i < 256; ++i) {
177       String str = "AA" + ((char)i) + "AA";
178       t.addRow("row" + i, str);
179     }
180 
181     db.close();
182   }
183 
184   public void x_testCreateAltIsoFile() throws Exception
185   {
186     Database db = openCopy(Database.FileFormat.V2000, new File("/tmp/test_ind.mdb"), true);
187 
188     Table t = db.getTable("Table1");
189 
190     for(int i = 0; i < 256; ++i) {
191       String str = "AA" + ((char)i) + "AA";
192       t.addRow("row" + i, str,
193                (byte)42 + i, (short)53 + i, 13 * i,
194                (6.7d / i), null, null, true);
195     }
196 
197     db.close();
198   }
199 
200   public void x_testWriteAllCodesMdb() throws Exception
201   {
202     Database db = create(Database.FileFormat.V2000, true);
203 
204 //     Table t = new TableBuilder("Table1")
205 //       .addColumn(new ColumnBuilder("key", DataType.TEXT))
206 //       .addColumn(new ColumnBuilder("data", DataType.TEXT))
207 //       .toTable(db);
208 
209 //     for(int i = 0; i <= 0xFFFF; ++i) {
210 //       // skip non-char chars
211 //       char c = (char)i;
212 //       if(Character.isHighSurrogate(c) || Character.isLowSurrogate(c)) {
213 //         continue;
214 //       }
215 //       String key = toUnicodeStr(c);
216 //       String str = "AA" + c + "AA";
217 //       t.addRow(key, str);
218 //     }
219 
220     Table t = new TableBuilder("Table5")
221       .addColumn(new ColumnBuilder("name", DataType.TEXT))
222       .addColumn(new ColumnBuilder("data", DataType.TEXT))
223       .toTable(db);
224 
225     char c = (char)0x3041;   // crazy 7F 02 ... A0
226     char c2 = (char)0x30A2;  // crazy 7F 02 ...
227     char c3 = (char)0x2045;  // inat 27 ... 1C
228     char c4 = (char)0x3043;  // crazy 7F 03 ... A0
229     char c5 = (char)0x3046;  // crazy 7F 04 ...
230     char c6 = (char)0x30F6;  // crazy 7F 0D ... A0
231     char c7 = (char)0x3099;  // unprint 03
232     char c8 = (char)0x0041;  // A
233     char c9 = (char)0x002D;  // - (unprint)
234     char c10 = (char)0x20E1; // unprint F2
235     char c11 = (char)0x309A; // unprint 04
236     char c12 = (char)0x01C4; // (long extra)
237     char c13 = (char)0x005F; // _ (long inline)
238     char c14 = (char)0xFFFE; // removed
239 
240     char[] cs = new char[]{c7, c8, c3, c12, c13, c14, c, c2, c9};
241     addCombos(t, 0, "", cs, 5);
242 
243 //     t = new TableBuilder("Table2")
244 //       .addColumn(new ColumnBuilder("data", DataType.TEXT))
245 //       .toTable(db);
246 
247 //     writeChars(0x0000, t);
248 
249 //     t = new TableBuilder("Table3")
250 //       .addColumn(new ColumnBuilder("data", DataType.TEXT))
251 //       .toTable(db);
252 
253 //     writeChars(0x0400, t);
254 
255 
256     db.close();
257   }
258 
259   public void x_testReadAllCodesMdb() throws Exception
260   {
261 //     Database db = openCopy(new File("/data2/jackcess_test/testAllIndexCodes.mdb"));
262 //     Database db = openCopy(new File("/data2/jackcess_test/testAllIndexCodes_orig.mdb"));
263 //     Database db = openCopy(new File("/data2/jackcess_test/testSomeMoreCodes.mdb"));
264     Database db = openCopy(Database.FileFormat.V2000, new File("/data2/jackcess_test/testStillMoreCodes.mdb"));
265     Table t = db.getTable("Table5");
266 
267     Index ind = t.getIndexes().iterator().next();
268     ((IndexImpl)ind).initialize();
269 
270     System.out.println("Ind " + ind);
271 
272     Cursor cursor = CursorBuilder.createCursor(ind);
273     while(cursor.moveToNextRow()) {
274       System.out.println("=======");
275       String entryStr =
276         entryToString(cursor.getSavepoint().getCurrentPosition());
277       System.out.println("Entry Bytes: " + entryStr);
278       System.out.println("Value: " + cursor.getCurrentRow() + "; " +
279                          toUnicodeStr(cursor.getCurrentRow().get("data")));
280     }
281 
282     db.close();
283   }
284 
285   private int addCombos(Table t, int rowNum, String s, char[] cs, int len)
286     throws Exception
287   {
288     if(s.length() >= len) {
289       return rowNum;
290     }
291 
292     for(int i = 0; i < cs.length; ++i) {
293       String name = "row" + (rowNum++);
294       String ss = s + cs[i];
295       t.addRow(name, ss);
296       rowNum = addCombos(t, rowNum, ss, cs, len);
297     }
298 
299     return rowNum;
300   }
301 
302   private void writeChars(int hibyte, Table t) throws Exception
303   {
304     char other = (char)(hibyte | 0x41);
305     for(int i = 0; i < 0xFF; ++i) {
306       char c = (char)(hibyte | i);
307       String str = "" + other + c + other;
308       t.addRow(str);
309     }
310   }
311 
312   public void x_testReadIsoMdb() throws Exception
313   {
314 //     Database db = open(new File("/tmp/test_ind.mdb"));
315 //     Database db = open(new File("/tmp/test_ind2.mdb"));
316     Database db = open(Database.FileFormat.V2000, new File("/tmp/test_ind3.mdb"));
317 //     Database db = open(new File("/tmp/test_ind4.mdb"));
318 
319     Table t = db.getTable("Table1");
320     Index index = t.getIndex("B");
321     ((IndexImpl)index).initialize();
322     System.out.println("Ind " + index);
323 
324     Cursor cursor = CursorBuilder.createCursor(index);
325     while(cursor.moveToNextRow()) {
326       System.out.println("=======");
327       System.out.println("Savepoint: " + cursor.getSavepoint());
328       System.out.println("Value: " + cursor.getCurrentRow());
329     }
330 
331     db.close();
332   }
333 
334   public void x_testReverseIsoMdb2010() throws Exception
335   {
336     Database db = open(Database.FileFormat.V2010, new File("/data2/jackcess_test/testAllIndexCodes3_2010.accdb"));
337 
338     Table t = db.getTable("Table1");
339     Index index = t.getIndexes().iterator().next();
340     ((IndexImpl)index).initialize();
341     System.out.println("Ind " + index);
342 
343     Pattern inlinePat = Pattern.compile("7F 0E 02 0E 02 (.*)0E 02 0E 02 01 00");
344     Pattern unprintPat = Pattern.compile("01 01 01 80 (.+) 06 (.+) 00");
345     Pattern unprint2Pat = Pattern.compile("0E 02 0E 02 0E 02 0E 02 01 02 (.+) 00");
346     Pattern inatPat = Pattern.compile("7F 0E 02 0E 02 (.*)0E 02 0E 02 01 02 02 (.+) 00");
347     Pattern inat2Pat = Pattern.compile("7F 0E 02 0E 02 (.*)0E 02 0E 02 01 (02 02 (.+))?01 01 (.*)FF 02 80 FF 80 00");
348 
349     Map<Character,String[]> inlineCodes = new TreeMap<Character,String[]>();
350     Map<Character,String[]> unprintCodes = new TreeMap<Character,String[]>();
351     Map<Character,String[]> unprint2Codes = new TreeMap<Character,String[]>();
352     Map<Character,String[]> inatInlineCodes = new TreeMap<Character,String[]>();
353     Map<Character,String[]> inatExtraCodes = new TreeMap<Character,String[]>();
354     Map<Character,String[]> inat2Codes = new TreeMap<Character,String[]>();
355     Map<Character,String[]> inat2ExtraCodes = new TreeMap<Character,String[]>();
356     Map<Character,String[]> inat2CrazyCodes = new TreeMap<Character,String[]>();
357 
358 
359     Cursor cursor = CursorBuilder.createCursor(index);
360     while(cursor.moveToNextRow()) {
361 //       System.out.println("=======");
362 //       System.out.println("Savepoint: " + cursor.getSavepoint());
363 //       System.out.println("Value: " + cursor.getCurrentRow());
364       Cursor.Savepoint savepoint = cursor.getSavepoint();
365       String entryStr = entryToString(savepoint.getCurrentPosition());
366 
367       Row row = cursor.getCurrentRow();
368       String value = row.getString("data");
369       String key = row.getString("key");
370       char c = value.charAt(2);
371 
372       System.out.println("=======");
373       System.out.println("RowId: " +
374                          savepoint.getCurrentPosition().getRowId());
375       System.out.println("Entry: " + entryStr);
376 //         System.out.println("Row: " + row);
377       System.out.println("Value: (" + key + ")" + value);
378       System.out.println("Char: " + c + ", " + (int)c + ", " +
379                          toUnicodeStr(c));
380 
381       String type = null;
382       if(entryStr.endsWith("01 00")) {
383 
384         // handle inline codes
385         type = "INLINE";
386         Matcher m = inlinePat.matcher(entryStr);
387         m.find();
388         handleInlineEntry(m.group(1), c, inlineCodes);
389 
390       } else if(entryStr.contains("01 01 01 80")) {
391 
392         // handle most unprintable codes
393         type = "UNPRINTABLE";
394         Matcher m = unprintPat.matcher(entryStr);
395         m.find();
396         handleUnprintableEntry(m.group(2), c, unprintCodes);
397 
398       } else if(entryStr.contains("01 02 02") &&
399                 !entryStr.contains("FF 02 80 FF 80")) {
400 
401         // handle chars w/ symbols
402         type = "CHAR_WITH_SYMBOL";
403         Matcher m = inatPat.matcher(entryStr);
404         m.find();
405         handleInternationalEntry(m.group(1), m.group(2), c,
406                                  inatInlineCodes, inatExtraCodes);
407 
408       } else if(entryStr.contains("0E 02 0E 02 0E 02 0E 02 01 02")) {
409 
410         // handle chars w/ symbols
411         type = "UNPRINTABLE_2";
412         Matcher m = unprint2Pat.matcher(entryStr);
413         m.find();
414         handleUnprintable2Entry(m.group(1), c, unprint2Codes);
415 
416       } else if(entryStr.contains("FF 02 80 FF 80")) {
417 
418         type = "CRAZY_INAT";
419         Matcher m = inat2Pat.matcher(entryStr);
420         m.find();
421         handleInternational2Entry(m.group(1), m.group(3), m.group(4), c,
422                                   inat2Codes, inat2ExtraCodes,
423                                   inat2CrazyCodes);
424 
425       } else {
426 
427         // throw new RuntimeException("unhandled " + entryStr);
428         System.out.println("unhandled " + entryStr);
429       }
430 
431       System.out.println("Type: " + type);
432     }
433 
434     System.out.println("\n***CODES");
435     for(int i = 0; i <= 0xFFFF; ++i) {
436 
437       if(i == 256) {
438         System.out.println("\n***EXTENDED CODES");
439       }
440 
441       // skip non-char chars
442       char c = (char)i;
443       if(Character.isHighSurrogate(c) || Character.isLowSurrogate(c)) {
444         continue;
445       }
446 
447       if(c == (char)0xFFFE) {
448         // this gets replaced with FFFD, treat it the same
449         c = (char)0xFFFD;
450       }
451 
452       Character cc = c;
453       String[] chars = inlineCodes.get(cc);
454       if(chars != null) {
455         if((chars.length == 1) && (chars[0].length() == 0)) {
456           System.out.println("X");
457         } else {
458           System.out.println("S" + toByteString(chars));
459         }
460         continue;
461       }
462 
463       chars = inatInlineCodes.get(cc);
464       if(chars != null) {
465         String[] extra = inatExtraCodes.get(cc);
466         System.out.println("I" + toByteString(chars) + "," +
467                            toByteString(extra));
468         continue;
469       }
470 
471       chars = unprintCodes.get(cc);
472       if(chars != null) {
473         System.out.println("U" + toByteString(chars));
474         continue;
475       }
476 
477       chars = unprint2Codes.get(cc);
478       if(chars != null) {
479         if(chars.length > 1) {
480           throw new RuntimeException("long unprint codes");
481         }
482         int val = Integer.parseInt(chars[0], 16) - 2;
483         String valStr = ByteUtil.toHexString(new byte[]{(byte)val}).trim();
484         System.out.println("P" + valStr);
485         continue;
486       }
487 
488       chars = inat2Codes.get(cc);
489       if(chars != null) {
490         String [] crazyCodes = inat2CrazyCodes.get(cc);
491         String crazyCode = "";
492         if(crazyCodes != null) {
493           if((crazyCodes.length != 1) || !"A0".equals(crazyCodes[0])) {
494             throw new RuntimeException("CC " + Arrays.asList(crazyCodes));
495           }
496           crazyCode = "1";
497         }
498 
499         String[] extra = inat2ExtraCodes.get(cc);
500         System.out.println("Z" + toByteString(chars) + "," +
501                            toByteString(extra) + "," +
502                            crazyCode);
503         continue;
504       }
505 
506       throw new RuntimeException("Unhandled char " + toUnicodeStr(c));
507     }
508     System.out.println("\n***END CODES");
509 
510     db.close();
511   }
512 
513   public void x_testReverseIsoMdb() throws Exception
514   {
515     Database db = open(Database.FileFormat.V2000, new File("/data2/jackcess_test/testAllIndexCodes3.mdb"));
516 
517     Table t = db.getTable("Table1");
518     Index index = t.getIndexes().iterator().next();
519     ((IndexImpl)index).initialize();
520     System.out.println("Ind " + index);
521 
522     Pattern inlinePat = Pattern.compile("7F 4A 4A (.*)4A 4A 01 00");
523     Pattern unprintPat = Pattern.compile("01 01 01 80 (.+) 06 (.+) 00");
524     Pattern unprint2Pat = Pattern.compile("4A 4A 4A 4A 01 02 (.+) 00");
525     Pattern inatPat = Pattern.compile("7F 4A 4A (.*)4A 4A 01 02 02 (.+) 00");
526     Pattern inat2Pat = Pattern.compile("7F 4A 4A (.*)4A 4A 01 (02 02 (.+))?01 01 (.*)FF 02 80 FF 80 00");
527 
528     Map<Character,String[]> inlineCodes = new TreeMap<Character,String[]>();
529     Map<Character,String[]> unprintCodes = new TreeMap<Character,String[]>();
530     Map<Character,String[]> unprint2Codes = new TreeMap<Character,String[]>();
531     Map<Character,String[]> inatInlineCodes = new TreeMap<Character,String[]>();
532     Map<Character,String[]> inatExtraCodes = new TreeMap<Character,String[]>();
533     Map<Character,String[]> inat2Codes = new TreeMap<Character,String[]>();
534     Map<Character,String[]> inat2ExtraCodes = new TreeMap<Character,String[]>();
535     Map<Character,String[]> inat2CrazyCodes = new TreeMap<Character,String[]>();
536 
537 
538     Cursor cursor = CursorBuilder.createCursor(index);
539     while(cursor.moveToNextRow()) {
540 //       System.out.println("=======");
541 //       System.out.println("Savepoint: " + cursor.getSavepoint());
542 //       System.out.println("Value: " + cursor.getCurrentRow());
543       Cursor.Savepoint savepoint = cursor.getSavepoint();
544       String entryStr = entryToString(savepoint.getCurrentPosition());
545 
546       Row row = cursor.getCurrentRow();
547       String value = row.getString("data");
548       String key = row.getString("key");
549       char c = value.charAt(2);
550       System.out.println("=======");
551       System.out.println("RowId: " +
552                          savepoint.getCurrentPosition().getRowId());
553       System.out.println("Entry: " + entryStr);
554 //         System.out.println("Row: " + row);
555       System.out.println("Value: (" + key + ")" + value);
556       System.out.println("Char: " + c + ", " + (int)c + ", " +
557                          toUnicodeStr(c));
558 
559       String type = null;
560       if(entryStr.endsWith("01 00")) {
561 
562         // handle inline codes
563         type = "INLINE";
564         Matcher m = inlinePat.matcher(entryStr);
565         m.find();
566         handleInlineEntry(m.group(1), c, inlineCodes);
567 
568       } else if(entryStr.contains("01 01 01 80")) {
569 
570         // handle most unprintable codes
571         type = "UNPRINTABLE";
572         Matcher m = unprintPat.matcher(entryStr);
573         m.find();
574         handleUnprintableEntry(m.group(2), c, unprintCodes);
575 
576       } else if(entryStr.contains("01 02 02") &&
577                 !entryStr.contains("FF 02 80 FF 80")) {
578 
579         // handle chars w/ symbols
580         type = "CHAR_WITH_SYMBOL";
581         Matcher m = inatPat.matcher(entryStr);
582         m.find();
583         handleInternationalEntry(m.group(1), m.group(2), c,
584                                  inatInlineCodes, inatExtraCodes);
585 
586       } else if(entryStr.contains("4A 4A 4A 4A 01 02")) {
587 
588         // handle chars w/ symbols
589         type = "UNPRINTABLE_2";
590         Matcher m = unprint2Pat.matcher(entryStr);
591         m.find();
592         handleUnprintable2Entry(m.group(1), c, unprint2Codes);
593 
594       } else if(entryStr.contains("FF 02 80 FF 80")) {
595 
596         type = "CRAZY_INAT";
597         Matcher m = inat2Pat.matcher(entryStr);
598         m.find();
599         handleInternational2Entry(m.group(1), m.group(3), m.group(4), c,
600                                   inat2Codes, inat2ExtraCodes,
601                                   inat2CrazyCodes);
602 
603       } else {
604 
605         throw new RuntimeException("unhandled " + entryStr);
606       }
607 
608       System.out.println("Type: " + type);
609     }
610 
611     System.out.println("\n***CODES");
612     for(int i = 0; i <= 0xFFFF; ++i) {
613 
614       if(i == 256) {
615         System.out.println("\n***EXTENDED CODES");
616       }
617 
618       // skip non-char chars
619       char c = (char)i;
620       if(Character.isHighSurrogate(c) || Character.isLowSurrogate(c)) {
621         continue;
622       }
623 
624       if(c == (char)0xFFFE) {
625         // this gets replaced with FFFD, treat it the same
626         c = (char)0xFFFD;
627       }
628 
629       Character cc = c;
630       String[] chars = inlineCodes.get(cc);
631       if(chars != null) {
632         if((chars.length == 1) && (chars[0].length() == 0)) {
633           System.out.println("X");
634         } else {
635           System.out.println("S" + toByteString(chars));
636         }
637         continue;
638       }
639 
640       chars = inatInlineCodes.get(cc);
641       if(chars != null) {
642         String[] extra = inatExtraCodes.get(cc);
643         System.out.println("I" + toByteString(chars) + "," +
644                            toByteString(extra));
645         continue;
646       }
647 
648       chars = unprintCodes.get(cc);
649       if(chars != null) {
650         System.out.println("U" + toByteString(chars));
651         continue;
652       }
653 
654       chars = unprint2Codes.get(cc);
655       if(chars != null) {
656         if(chars.length > 1) {
657           throw new RuntimeException("long unprint codes");
658         }
659         int val = Integer.parseInt(chars[0], 16) - 2;
660         String valStr = ByteUtil.toHexString(new byte[]{(byte)val}).trim();
661         System.out.println("P" + valStr);
662         continue;
663       }
664 
665       chars = inat2Codes.get(cc);
666       if(chars != null) {
667         String [] crazyCodes = inat2CrazyCodes.get(cc);
668         String crazyCode = "";
669         if(crazyCodes != null) {
670           if((crazyCodes.length != 1) || !"A0".equals(crazyCodes[0])) {
671             throw new RuntimeException("CC " + Arrays.asList(crazyCodes));
672           }
673           crazyCode = "1";
674         }
675 
676         String[] extra = inat2ExtraCodes.get(cc);
677         System.out.println("Z" + toByteString(chars) + "," +
678                            toByteString(extra) + "," +
679                            crazyCode);
680         continue;
681       }
682 
683       throw new RuntimeException("Unhandled char " + toUnicodeStr(c));
684     }
685     System.out.println("\n***END CODES");
686 
687     db.close();
688   }
689 
690   private static String toByteString(String[] chars)
691   {
692     String str = join(chars, "", "");
693     if(str.length() > 0 && str.charAt(0) == '0') {
694       str = str.substring(1);
695     }
696     return str;
697   }
698 
699   private static void handleInlineEntry(
700       String entryCodes, char c, Map<Character,String[]> inlineCodes)
701     throws Exception
702   {
703     inlineCodes.put(c, entryCodes.trim().split(" "));
704   }
705 
706   private static void handleUnprintableEntry(
707       String entryCodes, char c, Map<Character,String[]> unprintCodes)
708     throws Exception
709   {
710     unprintCodes.put(c, entryCodes.trim().split(" "));
711   }
712 
713   private static void handleUnprintable2Entry(
714       String entryCodes, char c, Map<Character,String[]> unprintCodes)
715     throws Exception
716   {
717     unprintCodes.put(c, entryCodes.trim().split(" "));
718   }
719 
720   private static void handleInternationalEntry(
721       String inlineCodes, String entryCodes, char c,
722       Map<Character,String[]> inatInlineCodes,
723       Map<Character,String[]> inatExtraCodes)
724     throws Exception
725   {
726     inatInlineCodes.put(c, inlineCodes.trim().split(" "));
727     inatExtraCodes.put(c, entryCodes.trim().split(" "));
728   }
729 
730   private static void handleInternational2Entry(
731       String inlineCodes, String entryCodes, String crazyCodes, char c,
732       Map<Character,String[]> inatInlineCodes,
733       Map<Character,String[]> inatExtraCodes,
734       Map<Character,String[]> inatCrazyCodes)
735     throws Exception
736   {
737     inatInlineCodes.put(c, inlineCodes.trim().split(" "));
738     if(entryCodes != null) {
739       inatExtraCodes.put(c, entryCodes.trim().split(" "));
740     }
741     if((crazyCodes != null) && (crazyCodes.length() > 0)) {
742       inatCrazyCodes.put(c, crazyCodes.trim().split(" "));
743     }
744   }
745 
746   public static String toUnicodeStr(Object obj) throws Exception {
747     StringBuilder sb = new StringBuilder();
748     for(char c : obj.toString().toCharArray()) {
749       sb.append(toUnicodeStr(c)).append(" ");
750     }
751     return sb.toString();
752   }
753 
754   private static String toUnicodeStr(char c) throws Exception {
755     String specialStr = SPECIAL_CHARS.get(c);
756     if(specialStr != null) {
757       return specialStr;
758     }
759 
760     String digits = Integer.toHexString(c).toUpperCase();
761     while(digits.length() < 4) {
762       digits = "0" + digits;
763     }
764     return "\\u" + digits;
765   }
766 
767   private static String join(String[] strs, String joinStr, String prefixStr) {
768     if(strs == null) {
769       return "";
770     }
771     StringBuilder builder = new StringBuilder();
772     for(int i = 0; i < strs.length; ++i) {
773       if(strs[i].length() == 0) {
774         continue;
775       }
776       builder.append(prefixStr).append(strs[i]);
777       if(i < (strs.length - 1)) {
778         builder.append(joinStr);
779       }
780     }
781     return builder.toString();
782   }
783 
784   public static String entryToString(Cursor.Position curPos)
785     throws Exception
786   {
787     Field eField = curPos.getClass().getDeclaredField("_entry");
788     eField.setAccessible(true);
789     IndexData.Entry entry = (IndexData.Entry)eField.get(curPos);
790     Field ebField = entry.getClass().getDeclaredField("_entryBytes");
791     ebField.setAccessible(true);
792     byte[] entryBytes = (byte[])ebField.get(entry);
793 
794     return ByteUtil.toHexString(ByteBuffer.wrap(entryBytes),
795                                 0, entryBytes.length, false);
796   }
797 
798 }