/* ******************************************************************************* * * Copyright (C) 2001-2005, International Business Machines * Corporation and others. All Rights Reserved. * ******************************************************************************* * file name: gennorm.c * encoding: US-ASCII * tab size: 8 (not used) * indentation:4 * * created on: 2001may25 * created by: Markus W. Scherer * * This program reads the Unicode character database text file, * parses it, and extracts the data for normalization. * It then preprocesses it and writes a binary file for efficient use * in various Unicode text normalization processes. */ #include <stdio.h> #include <stdlib.h> #include "unicode/utypes.h" #include "unicode/uchar.h" #include "unicode/ustring.h" #include "unicode/putil.h" #include "unicode/uclean.h" #include "unicode/udata.h" #include "unicode/uset.h" #include "cmemory.h" #include "cstring.h" #include "unewdata.h" #include "uoptions.h" #include "uparse.h" #include "unormimp.h" U_CDECL_BEGIN #include "gennorm.h" U_CDECL_END UBool beVerbose=FALSE, haveCopyright=TRUE; /* prototypes --------------------------------------------------------------- */ static void parseDerivedNormalizationProperties(const char *filename, UErrorCode *pErrorCode, UBool reportError); static void parseDB(const char *filename, UErrorCode *pErrorCode); /* -------------------------------------------------------------------------- */ enum { HELP_H, HELP_QUESTION_MARK, VERBOSE, COPYRIGHT, DESTDIR, SOURCEDIR, UNICODE_VERSION, ICUDATADIR, CSOURCE, STORE_FLAGS }; static UOption options[]={ UOPTION_HELP_H, UOPTION_HELP_QUESTION_MARK, UOPTION_VERBOSE, UOPTION_COPYRIGHT, UOPTION_DESTDIR, UOPTION_SOURCEDIR, UOPTION_DEF("unicode", 'u', UOPT_REQUIRES_ARG), UOPTION_ICUDATADIR, UOPTION_DEF("csource", 'C', UOPT_NO_ARG), UOPTION_DEF("prune", 'p', UOPT_REQUIRES_ARG) }; extern int main(int argc, char* argv[]) { #if !UCONFIG_NO_NORMALIZATION char filename[300]; #endif const char *srcDir=NULL, *destDir=NULL, *suffix=NULL; char *basename=NULL; UErrorCode errorCode=U_ZERO_ERROR; U_MAIN_INIT_ARGS(argc, argv); /* preset then read command line options */ options[4].value=u_getDataDirectory(); options[5].value=""; options[6].value="3.0.0"; options[ICUDATADIR].value=u_getDataDirectory(); argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options); /* error handling, printing usage message */ if(argc<0) { fprintf(stderr, "error in command line argument \"%s\"\n", argv[-argc]); } if(argc<0 || options[0].doesOccur || options[1].doesOccur) { /* * Broken into chucks because the C89 standard says the minimum * required supported string length is 509 bytes. */ fprintf(stderr, "Usage: %s [-options] [suffix]\n" "\n" "Read the UnicodeData.txt file and other Unicode properties files and\n" "create a binary file " U_ICUDATA_NAME "_" DATA_NAME "." DATA_TYPE " with the normalization data\n" "\n", argv[0]); fprintf(stderr, "Options:\n" "\t-h or -? or --help this usage text\n" "\t-v or --verbose verbose output\n" "\t-c or --copyright include a copyright notice\n" "\t-u or --unicode Unicode version, followed by the version like 3.0.0\n" "\t-C or --csource generate a .c source file rather than the .icu binary\n"); fprintf(stderr, "\t-p or --prune flags Prune for data modularization:\n" "\t Determine what data is to be stored.\n" "\t 0 (zero) stores minimal data (only for NFD)\n" "\t lowercase letters turn off data, uppercase turn on (use with 0)\n"); fprintf(stderr, "\t k: compatibility decompositions (NFKC, NFKD)\n" "\t c: composition data (NFC, NFKC)\n" "\t f: FCD data (will be generated at load time)\n" "\t a: auxiliary data (canonical closure etc.)\n" "\t x: exclusion sets (Unicode 3.2-level normalization)\n"); fprintf(stderr, "\t-d or --destdir destination directory, followed by the path\n" "\t-s or --sourcedir source directory, followed by the path\n" "\t-i or --icudatadir directory for locating any needed intermediate data files,\n" "\t followed by path, defaults to <%s>\n" "\tsuffix suffix that is to be appended with a '-'\n" "\t to the source file basenames before opening;\n" "\t 'gennorm new' will read UnicodeData-new.txt etc.\n", u_getDataDirectory()); return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR; } /* get the options values */ beVerbose=options[2].doesOccur; haveCopyright=options[3].doesOccur; srcDir=options[5].value; destDir=options[4].value; if(argc>=2) { suffix=argv[1]; } else { suffix=NULL; } #if UCONFIG_NO_NORMALIZATION fprintf(stderr, "gennorm writes a dummy " U_ICUDATA_NAME "_" DATA_NAME "." DATA_TYPE " because UCONFIG_NO_NORMALIZATION is set, \n" "see icu/source/common/unicode/uconfig.h\n"); generateData(destDir, options[CSOURCE].doesOccur); #else setUnicodeVersion(options[6].value); if (options[ICUDATADIR].doesOccur) { u_setDataDirectory(options[ICUDATADIR].value); } if(options[STORE_FLAGS].doesOccur) { const char *s=options[STORE_FLAGS].value; char c; while((c=*s++)!=0) { switch(c) { case '0': gStoreFlags=0; /* store minimal data (only for NFD) */ break; /* lowercase letters: omit data */ case 'k': gStoreFlags&=~U_MASK(UGENNORM_STORE_COMPAT); break; case 'c': gStoreFlags&=~U_MASK(UGENNORM_STORE_COMPOSITION); break; case 'f': gStoreFlags&=~U_MASK(UGENNORM_STORE_FCD); break; case 'a': gStoreFlags&=~U_MASK(UGENNORM_STORE_AUX); break; case 'x': gStoreFlags&=~U_MASK(UGENNORM_STORE_EXCLUSIONS); break; /* uppercase letters: include data (use with 0) */ case 'K': gStoreFlags|=U_MASK(UGENNORM_STORE_COMPAT); break; case 'C': gStoreFlags|=U_MASK(UGENNORM_STORE_COMPOSITION); break; case 'F': gStoreFlags|=U_MASK(UGENNORM_STORE_FCD); break; case 'A': gStoreFlags|=U_MASK(UGENNORM_STORE_AUX); break; case 'X': gStoreFlags|=U_MASK(UGENNORM_STORE_EXCLUSIONS); break; default: fprintf(stderr, "ignoring undefined prune flag '%c'\n", c); break; } } } /* * Verify that we can work with properties * but don't call u_init() because that needs unorm.icu which we are just * going to build here. */ { U_STRING_DECL(ideo, "[:Ideographic:]", 15); USet *set; U_STRING_INIT(ideo, "[:Ideographic:]", 15); set=uset_openPattern(ideo, -1, &errorCode); if(U_FAILURE(errorCode) || !uset_contains(set, 0xf900)) { fprintf(stderr, "gennorm is unable to work with properties (uprops.icu): %s\n", u_errorName(errorCode)); exit(errorCode); } uset_close(set); } /* prepare the filename beginning with the source dir */ uprv_strcpy(filename, srcDir); basename=filename+uprv_strlen(filename); if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) { *basename++=U_FILE_SEP_CHAR; } /* initialize */ init(); /* process DerivedNormalizationProps.txt (name changed for Unicode 3.2, to <=31 characters) */ if(suffix==NULL) { uprv_strcpy(basename, "DerivedNormalizationProps.txt"); } else { uprv_strcpy(basename, "DerivedNormalizationProps"); basename[30]='-'; uprv_strcpy(basename+31, suffix); uprv_strcat(basename+31, ".txt"); } parseDerivedNormalizationProperties(filename, &errorCode, FALSE); if(U_FAILURE(errorCode)) { /* can be only U_FILE_ACCESS_ERROR - try filename from before Unicode 3.2 */ if(suffix==NULL) { uprv_strcpy(basename, "DerivedNormalizationProperties.txt"); } else { uprv_strcpy(basename, "DerivedNormalizationProperties"); basename[30]='-'; uprv_strcpy(basename+31, suffix); uprv_strcat(basename+31, ".txt"); } parseDerivedNormalizationProperties(filename, &errorCode, TRUE); } /* process UnicodeData.txt */ if(suffix==NULL) { uprv_strcpy(basename, "UnicodeData.txt"); } else { uprv_strcpy(basename, "UnicodeData"); basename[11]='-'; uprv_strcpy(basename+12, suffix); uprv_strcat(basename+12, ".txt"); } parseDB(filename, &errorCode); /* process parsed data */ if(U_SUCCESS(errorCode)) { processData(); /* write the properties data file */ generateData(destDir, options[CSOURCE].doesOccur); cleanUpData(); } #endif return errorCode; } #if !UCONFIG_NO_NORMALIZATION /* parser for DerivedNormalizationProperties.txt ---------------------------- */ static void U_CALLCONV derivedNormalizationPropertiesLineFn(void *context, char *fields[][2], int32_t fieldCount, UErrorCode *pErrorCode) { UChar string[32]; char *s; uint32_t start, end; int32_t count; uint8_t qcFlags; /* get code point range */ count=u_parseCodePointRange(fields[0][0], &start, &end, pErrorCode); if(U_FAILURE(*pErrorCode)) { fprintf(stderr, "gennorm: error parsing DerivedNormalizationProperties.txt mapping at %s\n", fields[0][0]); exit(*pErrorCode); } /* ignore hangul - handle explicitly */ if(start==0xac00) { return; } /* get property - ignore unrecognized ones */ s=(char *)u_skipWhitespace(fields[1][0]); if(*s=='N' && s[1]=='F') { /* quick check flag */ qcFlags=0x11; s+=2; if(*s=='K') { qcFlags<<=1; ++s; } if(*s=='C' && s[1]=='_') { s+=2; } else if(*s=='D' && s[1]=='_') { qcFlags<<=2; s+=2; } else { return; } if(0==uprv_strncmp(s, "NO", 2)) { qcFlags&=0xf; } else if(0==uprv_strncmp(s, "MAYBE", 5)) { qcFlags&=0x30; } else if(0==uprv_strncmp(s, "QC", 2) && *(s=(char *)u_skipWhitespace(s+2))==';') { /* * Unicode 4.0.1: * changes single field "NFD_NO" -> two fields "NFD_QC; N" etc. */ /* start of the field */ s=(char *)u_skipWhitespace(s+1); if(*s=='N') { qcFlags&=0xf; } else if(*s=='M') { qcFlags&=0x30; } else { return; /* do nothing for "Yes" because it's the default value */ } } else { return; /* do nothing for "Yes" because it's the default value */ } /* set this flag for all code points in this range */ while(start<=end) { setQCFlags(start++, qcFlags); } } else if(0==uprv_memcmp(s, "Comp_Ex", 7) || 0==uprv_memcmp(s, "Full_Composition_Exclusion", 26)) { /* full composition exclusion */ while(start<=end) { setCompositionExclusion(start++); } } else if( ((0==uprv_memcmp(s, "FNC", 3) && *(s=(char *)u_skipWhitespace(s+3))==';') || (0==uprv_memcmp(s, "FC_NFKC", 7) && *(s=(char *)u_skipWhitespace(s+7))==';')) ) { /* FC_NFKC_Closure, parse field 2 to get the string */ char *t; /* start of the field */ s=(char *)u_skipWhitespace(s+1); /* find the end of the field */ for(t=s; *t!=';' && *t!='#' && *t!=0 && *t!='\n' && *t!='\r'; ++t) {} *t=0; string[0]=(UChar)u_parseString(s, string+1, 31, NULL, pErrorCode); if(U_FAILURE(*pErrorCode)) { fprintf(stderr, "gennorm error: illegal FNC string at %s\n", fields[0][0]); exit(*pErrorCode); } while(start<=end) { setFNC(start++, string); } } } static void parseDerivedNormalizationProperties(const char *filename, UErrorCode *pErrorCode, UBool reportError) { char *fields[2][2]; if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { return; } u_parseDelimitedFile(filename, ';', fields, 2, derivedNormalizationPropertiesLineFn, NULL, pErrorCode); if(U_FAILURE(*pErrorCode) && (reportError || *pErrorCode!=U_FILE_ACCESS_ERROR)) { fprintf(stderr, "gennorm error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode)); exit(*pErrorCode); } } /* parser for UnicodeData.txt ----------------------------------------------- */ static void U_CALLCONV unicodeDataLineFn(void *context, char *fields[][2], int32_t fieldCount, UErrorCode *pErrorCode) { uint32_t decomp[40]; Norm norm; const char *s; char *end; uint32_t code, value; int32_t length; UBool isCompat, something=FALSE; /* ignore First and Last entries for ranges */ if( *fields[1][0]=='<' && (length=(int32_t)(fields[1][1]-fields[1][0]))>=9 && (0==uprv_memcmp(", First>", fields[1][1]-8, 8) || 0==uprv_memcmp(", Last>", fields[1][1]-7, 7)) ) { return; } /* reset the properties */ uprv_memset(&norm, 0, sizeof(Norm)); /* * The combiningIndex must not be initialized to 0 because 0 is the * combiningIndex of the first forward-combining character. */ norm.combiningIndex=0xffff; /* get the character code, field 0 */ code=(uint32_t)uprv_strtoul(fields[0][0], &end, 16); if(end<=fields[0][0] || end!=fields[0][1]) { fprintf(stderr, "gennorm: syntax error in field 0 at %s\n", fields[0][0]); *pErrorCode=U_PARSE_ERROR; exit(U_PARSE_ERROR); } /* get canonical combining class, field 3 */ value=(uint32_t)uprv_strtoul(fields[3][0], &end, 10); if(end<=fields[3][0] || end!=fields[3][1] || value>0xff) { fprintf(stderr, "gennorm: syntax error in field 3 at %s\n", fields[0][0]); *pErrorCode=U_PARSE_ERROR; exit(U_PARSE_ERROR); } if(value>0) { norm.udataCC=(uint8_t)value; something=TRUE; } /* get the decomposition, field 5 */ if(fields[5][0]<fields[5][1]) { if(*(s=fields[5][0])=='<') { ++s; isCompat=TRUE; /* skip and ignore the compatibility type name */ do { if(s==fields[5][1]) { /* missing '>' */ fprintf(stderr, "gennorm: syntax error in field 5 at %s\n", fields[0][0]); *pErrorCode=U_PARSE_ERROR; exit(U_PARSE_ERROR); } } while(*s++!='>'); } else { isCompat=FALSE; } /* parse the decomposition string */ length=u_parseCodePoints(s, decomp, sizeof(decomp)/4, pErrorCode); if(U_FAILURE(*pErrorCode)) { fprintf(stderr, "gennorm error parsing UnicodeData.txt decomposition of U+%04lx - %s\n", (long)code, u_errorName(*pErrorCode)); exit(*pErrorCode); } /* store the string */ if(length>0) { something=TRUE; if(isCompat) { norm.lenNFKD=(uint8_t)length; norm.nfkd=decomp; } else { if(length>2) { fprintf(stderr, "gennorm: error - length of NFD(U+%04lx) = %ld >2 in UnicodeData - illegal\n", (long)code, (long)length); *pErrorCode=U_PARSE_ERROR; exit(U_PARSE_ERROR); } norm.lenNFD=(uint8_t)length; norm.nfd=decomp; } } } /* check for non-character code points */ if((code&0xfffe)==0xfffe || (uint32_t)(code-0xfdd0)<0x20 || code>0x10ffff) { fprintf(stderr, "gennorm: error - properties for non-character code point U+%04lx\n", (long)code); *pErrorCode=U_PARSE_ERROR; exit(U_PARSE_ERROR); } if(something) { /* there are normalization values, so store them */ #if 0 if(beVerbose) { printf("store values for U+%04lx: cc=%d, lenNFD=%ld, lenNFKD=%ld\n", (long)code, norm.udataCC, (long)norm.lenNFD, (long)norm.lenNFKD); } #endif storeNorm(code, &norm); } } static void parseDB(const char *filename, UErrorCode *pErrorCode) { char *fields[15][2]; if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { return; } u_parseDelimitedFile(filename, ';', fields, 15, unicodeDataLineFn, NULL, pErrorCode); if(U_FAILURE(*pErrorCode)) { fprintf(stderr, "gennorm error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode)); exit(*pErrorCode); } } #endif /* #if !UCONFIG_NO_NORMALIZATION */ /* * Hey, Emacs, please set the following: * * Local Variables: * indent-tabs-mode: nil * End: * */