// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
******************************************************************************
*
* Copyright (C) 1999-2016, International Business Machines
* Corporation and others. All Rights Reserved.
*
******************************************************************************
* file name: udata.cpp
* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
* created on: 1999oct25
* created by: Markus W. Scherer
*/
#include "unicode/utypes.h" /* U_PLATFORM etc. */
#ifdef __GNUC__
/* if gcc
#define ATTRIBUTE_WEAK __attribute__ ((weak))
might have to #include some other header
*/
#endif
#include "unicode/putil.h"
#include "unicode/udata.h"
#include "unicode/uversion.h"
#include "charstr.h"
#include "cmemory.h"
#include "cstring.h"
#include "mutex.h"
#include "putilimp.h"
#include "uassert.h"
#include "ucln_cmn.h"
#include "ucmndata.h"
#include "udatamem.h"
#include "uhash.h"
#include "umapfile.h"
#include "umutex.h"
/***********************************************************************
*
* Notes on the organization of the ICU data implementation
*
* All of the public API is defined in udata.h
*
* The implementation is split into several files...
*
* - udata.c (this file) contains higher level code that knows about
* the search paths for locating data, caching opened data, etc.
*
* - umapfile.c contains the low level platform-specific code for actually loading
* (memory mapping, file reading, whatever) data into memory.
*
* - ucmndata.c deals with the tables of contents of ICU data items within
* an ICU common format data file. The implementation includes
* an abstract interface and support for multiple TOC formats.
* All knowledge of any specific TOC format is encapsulated here.
*
* - udatamem.c has code for managing UDataMemory structs. These are little
* descriptor objects for blocks of memory holding ICU data of
* various types.
*/
/* configuration ---------------------------------------------------------- */
/* If you are excruciatingly bored turn this on .. */
/* #define UDATA_DEBUG 1 */
#if defined(UDATA_DEBUG)
# include <stdio.h>
#endif
U_NAMESPACE_USE
/*
* Forward declarations
*/
static UDataMemory *udata_findCachedData(const char *path, UErrorCode &err);
/***********************************************************************
*
* static (Global) data
*
************************************************************************/
/*
* Pointers to the common ICU data.
*
* We store multiple pointers to ICU data packages and iterate through them
* when looking for a data item.
*
* It is possible to combine this with dependency inversion:
* One or more data package libraries may export
* functions that each return a pointer to their piece of the ICU data,
* and this file would import them as weak functions, without a
* strong linker dependency from the common library on the data library.
*
* Then we can have applications depend on only that part of ICU's data
* that they really need, reducing the size of binaries that take advantage
* of this.
*/
static UDataMemory *gCommonICUDataArray[10] = { NULL }; // Access protected by icu global mutex.
static u_atomic_int32_t gHaveTriedToLoadCommonData = ATOMIC_INT32_T_INITIALIZER(0); // See extendICUData().
static UHashtable *gCommonDataCache = NULL; /* Global hash table of opened ICU data files. */
static icu::UInitOnce gCommonDataCacheInitOnce = U_INITONCE_INITIALIZER;
#if U_PLATFORM_HAS_WINUWP_API == 0
static UDataFileAccess gDataFileAccess = UDATA_DEFAULT_ACCESS; // Access not synchronized.
// Modifying is documented as thread-unsafe.
#else
static UDataFileAccess gDataFileAccess = UDATA_NO_FILES; // Windows UWP looks in one spot explicitly
#endif
static UBool U_CALLCONV
udata_cleanup(void)
{
int32_t i;
if (gCommonDataCache) { /* Delete the cache of user data mappings. */
uhash_close(gCommonDataCache); /* Table owns the contents, and will delete them. */
gCommonDataCache = NULL; /* Cleanup is not thread safe. */
}
gCommonDataCacheInitOnce.reset();
for (i = 0; i < UPRV_LENGTHOF(gCommonICUDataArray) && gCommonICUDataArray[i] != NULL; ++i) {
udata_close(gCommonICUDataArray[i]);
gCommonICUDataArray[i] = NULL;
}
gHaveTriedToLoadCommonData = 0;
return TRUE; /* Everything was cleaned up */
}
static UBool U_CALLCONV
findCommonICUDataByName(const char *inBasename, UErrorCode &err)
{
UBool found = FALSE;
int32_t i;
UDataMemory *pData = udata_findCachedData(inBasename, err);
if (U_FAILURE(err) || pData == NULL)
return FALSE;
{
Mutex lock;
for (i = 0; i < UPRV_LENGTHOF(gCommonICUDataArray); ++i) {
if ((gCommonICUDataArray[i] != NULL) && (gCommonICUDataArray[i]->pHeader == pData->pHeader)) {
/* The data pointer is already in the array. */
found = TRUE;
break;
}
}
}
return found;
}
/*
* setCommonICUData. Set a UDataMemory to be the global ICU Data
*/
static UBool
setCommonICUData(UDataMemory *pData, /* The new common data. Belongs to caller, we copy it. */
UBool warn, /* If true, set USING_DEFAULT warning if ICUData was */
/* changed by another thread before we got to it. */
UErrorCode *pErr)
{
UDataMemory *newCommonData = UDataMemory_createNewInstance(pErr);
int32_t i;
UBool didUpdate = FALSE;
if (U_FAILURE(*pErr)) {
return FALSE;
}
/* For the assignment, other threads must cleanly see either the old */
/* or the new, not some partially initialized new. The old can not be */
/* deleted - someone may still have a pointer to it lying around in */
/* their locals. */
UDatamemory_assign(newCommonData, pData);
umtx_lock(NULL);
for (i = 0; i < UPRV_LENGTHOF(gCommonICUDataArray); ++i) {
if (gCommonICUDataArray[i] == NULL) {
gCommonICUDataArray[i] = newCommonData;
didUpdate = TRUE;
break;
} else if (gCommonICUDataArray[i]->pHeader == pData->pHeader) {
/* The same data pointer is already in the array. */
break;
}
}
umtx_unlock(NULL);
if (i == UPRV_LENGTHOF(gCommonICUDataArray) && warn) {
*pErr = U_USING_DEFAULT_WARNING;
}
if (didUpdate) {
ucln_common_registerCleanup(UCLN_COMMON_UDATA, udata_cleanup);
} else {
uprv_free(newCommonData);
}
return didUpdate;
}
#if U_PLATFORM_HAS_WINUWP_API == 0
static UBool
setCommonICUDataPointer(const void *pData, UBool /*warn*/, UErrorCode *pErrorCode) {
UDataMemory tData;
UDataMemory_init(&tData);
UDataMemory_setData(&tData, pData);
udata_checkCommonData(&tData, pErrorCode);
return setCommonICUData(&tData, FALSE, pErrorCode);
}
#endif
static const char *
findBasename(const char *path) {
const char *basename=uprv_strrchr(path, U_FILE_SEP_CHAR);
if(basename==NULL) {
return path;
} else {
return basename+1;
}
}
#ifdef UDATA_DEBUG
static const char *
packageNameFromPath(const char *path)
{
if((path == NULL) || (*path == 0)) {
return U_ICUDATA_NAME;
}
path = findBasename(path);
if((path == NULL) || (*path == 0)) {
return U_ICUDATA_NAME;
}
return path;
}
#endif
/*----------------------------------------------------------------------*
* *
* Cache for common data *
* Functions for looking up or adding entries to a cache of *
* data that has been previously opened. Avoids a potentially *
* expensive operation of re-opening the data for subsequent *
* uses. *
* *
* Data remains cached for the duration of the process. *
* *
*----------------------------------------------------------------------*/
typedef struct DataCacheElement {
char *name;
UDataMemory *item;
} DataCacheElement;
/*
* Deleter function for DataCacheElements.
* udata cleanup function closes the hash table; hash table in turn calls back to
* here for each entry.
*/
static void U_CALLCONV DataCacheElement_deleter(void *pDCEl) {
DataCacheElement *p = (DataCacheElement *)pDCEl;
udata_close(p->item); /* unmaps storage */
uprv_free(p->name); /* delete the hash key string. */
uprv_free(pDCEl); /* delete 'this' */
}
static void U_CALLCONV udata_initHashTable(UErrorCode &err) {
U_ASSERT(gCommonDataCache == NULL);
gCommonDataCache = uhash_open(uhash_hashChars, uhash_compareChars, NULL, &err);
if (U_FAILURE(err)) {
return;
}
U_ASSERT(gCommonDataCache != NULL);
uhash_setValueDeleter(gCommonDataCache, DataCacheElement_deleter);
ucln_common_registerCleanup(UCLN_COMMON_UDATA, udata_cleanup);
}
/* udata_getCacheHashTable()
* Get the hash table used to store the data cache entries.
* Lazy create it if it doesn't yet exist.
*/
static UHashtable *udata_getHashTable(UErrorCode &err) {
umtx_initOnce(gCommonDataCacheInitOnce, &udata_initHashTable, err);
return gCommonDataCache;
}
static UDataMemory *udata_findCachedData(const char *path, UErrorCode &err)
{
UHashtable *htable;
UDataMemory *retVal = NULL;
DataCacheElement *el;
const char *baseName;
htable = udata_getHashTable(err);
if (U_FAILURE(err)) {
return NULL;
}
baseName = findBasename(path); /* Cache remembers only the base name, not the full path. */
umtx_lock(NULL);
el = (DataCacheElement *)uhash_get(htable, baseName);
umtx_unlock(NULL);
if (el != NULL) {
retVal = el->item;
}
#ifdef UDATA_DEBUG
fprintf(stderr, "Cache: [%s] -> %p\n", baseName, retVal);
#endif
return retVal;
}
static UDataMemory *udata_cacheDataItem(const char *path, UDataMemory *item, UErrorCode *pErr) {
DataCacheElement *newElement;
const char *baseName;
int32_t nameLen;
UHashtable *htable;
DataCacheElement *oldValue = NULL;
UErrorCode subErr = U_ZERO_ERROR;
htable = udata_getHashTable(*pErr);
if (U_FAILURE(*pErr)) {
return NULL;
}
/* Create a new DataCacheElement - the thingy we store in the hash table -
* and copy the supplied path and UDataMemoryItems into it.
*/
newElement = (DataCacheElement *)uprv_malloc(sizeof(DataCacheElement));
if (newElement == NULL) {
*pErr = U_MEMORY_ALLOCATION_ERROR;
return NULL;
}
newElement->item = UDataMemory_createNewInstance(pErr);
if (U_FAILURE(*pErr)) {
uprv_free(newElement);
return NULL;
}
UDatamemory_assign(newElement->item, item);
baseName = findBasename(path);
nameLen = (int32_t)uprv_strlen(baseName);
newElement->name = (char *)uprv_malloc(nameLen+1);
if (newElement->name == NULL) {
*pErr = U_MEMORY_ALLOCATION_ERROR;
uprv_free(newElement->item);
uprv_free(newElement);
return NULL;
}
uprv_strcpy(newElement->name, baseName);
/* Stick the new DataCacheElement into the hash table.
*/
umtx_lock(NULL);
oldValue = (DataCacheElement *)uhash_get(htable, path);
if (oldValue != NULL) {
subErr = U_USING_DEFAULT_WARNING;
}
else {
uhash_put(
htable,
newElement->name, /* Key */
newElement, /* Value */
&subErr);
}
umtx_unlock(NULL);
#ifdef UDATA_DEBUG
fprintf(stderr, "Cache: [%s] <<< %p : %s. vFunc=%p\n", newElement->name,
newElement->item, u_errorName(subErr), newElement->item->vFuncs);
#endif
if (subErr == U_USING_DEFAULT_WARNING || U_FAILURE(subErr)) {
*pErr = subErr; /* copy sub err unto fillin ONLY if something happens. */
uprv_free(newElement->name);
uprv_free(newElement->item);
uprv_free(newElement);
return oldValue ? oldValue->item : NULL;
}
return newElement->item;
}
/*----------------------------------------------------------------------*==============
* *
* Path management. Could be shared with other tools/etc if need be *
* later on. *
* *
*----------------------------------------------------------------------*/
U_NAMESPACE_BEGIN
class UDataPathIterator
{
public:
UDataPathIterator(const char *path, const char *pkg,
const char *item, const char *suffix, UBool doCheckLastFour,
UErrorCode *pErrorCode);
const char *next(UErrorCode *pErrorCode);
private:
const char *path; /* working path (u_icudata_Dir) */
const char *nextPath; /* path following this one */
const char *basename; /* item's basename (icudt22e_mt.res)*/
StringPiece suffix; /* item suffix (can be null) */
uint32_t basenameLen; /* length of basename */
CharString itemPath; /* path passed in with item name */
CharString pathBuffer; /* output path for this it'ion */
CharString packageStub; /* example: "/icudt28b". Will ignore that leaf in set paths. */
UBool checkLastFour; /* if TRUE then allow paths such as '/foo/myapp.dat'
* to match, checks last 4 chars of suffix with
* last 4 of path, then previous chars. */
};
/**
* @param iter The iterator to be initialized. Its current state does not matter.
* @param inPath The full pathname to be iterated over. If NULL, defaults to U_ICUDATA_NAME
* @param pkg Package which is being searched for, ex "icudt28l". Will ignore leaf directories such as /icudt28l
* @param item Item to be searched for. Can include full path, such as /a/b/foo.dat
* @param inSuffix Optional item suffix, if not-null (ex. ".dat") then 'path' can contain 'item' explicitly.
* Ex: 'stuff.dat' would be found in '/a/foo:/tmp/stuff.dat:/bar/baz' as item #2.
* '/blarg/stuff.dat' would also be found.
* Note: inSuffix may also be the 'item' being searched for as well, (ex: "ibm-5348_P100-1997.cnv"), in which case
* the 'item' parameter is often the same as pkg. (Though sometimes might have a tree part as well, ex: "icudt62l-curr").
*/
UDataPathIterator::UDataPathIterator(const char *inPath, const char *pkg,
const char *item, const char *inSuffix, UBool doCheckLastFour,
UErrorCode *pErrorCode)
{
#ifdef UDATA_DEBUG
fprintf(stderr, "SUFFIX1=%s PATH=%s\n", inSuffix, inPath);
#endif
/** Path **/
if(inPath == NULL) {
path = u_getDataDirectory();
} else {
path = inPath;
}
/** Package **/
if(pkg != NULL) {
packageStub.append(U_FILE_SEP_CHAR, *pErrorCode).append(pkg, *pErrorCode);
#ifdef UDATA_DEBUG
fprintf(stderr, "STUB=%s [%d]\n", packageStub.data(), packageStub.length());
#endif
}
/** Item **/
basename = findBasename(item);
basenameLen = (int32_t)uprv_strlen(basename);
/** Item path **/
if(basename == item) {
nextPath = path;
} else {
itemPath.append(item, (int32_t)(basename-item), *pErrorCode);
nextPath = itemPath.data();
}
#ifdef UDATA_DEBUG
fprintf(stderr, "SUFFIX=%s [%p]\n", inSuffix, inSuffix);
#endif
/** Suffix **/
if(inSuffix != NULL) {
suffix = inSuffix;
} else {
suffix = "";
}
checkLastFour = doCheckLastFour;
/* pathBuffer will hold the output path strings returned by this iterator */
#ifdef UDATA_DEBUG
fprintf(stderr, "%p: init %s -> [path=%s], [base=%s], [suff=%s], [itempath=%s], [nextpath=%s], [checklast4=%s]\n",
iter,
item,
path,
basename,
suffix,
itemPath.data(),
nextPath,
checkLastFour?"TRUE":"false");
#endif
}
/**
* Get the next path on the list.
*
* @param iter The Iter to be used
* @param len If set, pointer to the length of the returned path, for convenience.
* @return Pointer to the next path segment, or NULL if there are no more.
*/
const char *UDataPathIterator::next(UErrorCode *pErrorCode)
{
if(U_FAILURE(*pErrorCode)) {
return NULL;
}
const char *currentPath = NULL;
int32_t pathLen = 0;
const char *pathBasename;
do
{
if( nextPath == NULL ) {
break;
}
currentPath = nextPath;
if(nextPath == itemPath.data()) { /* we were processing item's path. */
nextPath = path; /* start with regular path next tm. */
pathLen = (int32_t)uprv_strlen(currentPath);
} else {
/* fix up next for next time */
nextPath = uprv_strchr(currentPath, U_PATH_SEP_CHAR);
if(nextPath == NULL) {
/* segment: entire path */
pathLen = (int32_t)uprv_strlen(currentPath);
} else {
/* segment: until next segment */
pathLen = (int32_t)(nextPath - currentPath);
/* skip divider */
nextPath ++;
}
}
if(pathLen == 0) {
continue;
}
#ifdef UDATA_DEBUG
fprintf(stderr, "rest of path (IDD) = %s\n", currentPath);
fprintf(stderr, " ");
{
uint32_t qqq;
for(qqq=0;qqq<pathLen;qqq++)
{
fprintf(stderr, " ");
}
fprintf(stderr, "^\n");
}
#endif
pathBuffer.clear().append(currentPath, pathLen, *pErrorCode);
/* check for .dat files */
pathBasename = findBasename(pathBuffer.data());
if(checkLastFour == TRUE &&
(pathLen>=4) &&
uprv_strncmp(pathBuffer.data() +(pathLen-4), suffix.data(), 4)==0 && /* suffix matches */
uprv_strncmp(findBasename(pathBuffer.data()), basename, basenameLen)==0 && /* base matches */
uprv_strlen(pathBasename)==(basenameLen+4)) { /* base+suffix = full len */
#ifdef UDATA_DEBUG
fprintf(stderr, "Have %s file on the path: %s\n", suffix, pathBuffer.data());
#endif
/* do nothing */
}
else
{ /* regular dir path */
if(pathBuffer[pathLen-1] != U_FILE_SEP_CHAR) {
if((pathLen>=4) &&
uprv_strncmp(pathBuffer.data()+(pathLen-4), ".dat", 4) == 0)
{
#ifdef UDATA_DEBUG
fprintf(stderr, "skipping non-directory .dat file %s\n", pathBuffer.data());
#endif
continue;
}
/* Check if it is a directory with the same name as our package */
if(!packageStub.isEmpty() &&
(pathLen > packageStub.length()) &&
!uprv_strcmp(pathBuffer.data() + pathLen - packageStub.length(), packageStub.data())) {
#ifdef UDATA_DEBUG
fprintf(stderr, "Found stub %s (will add package %s of len %d)\n", packageStub.data(), basename, basenameLen);
#endif
pathBuffer.truncate(pathLen - packageStub.length());
}
pathBuffer.append(U_FILE_SEP_CHAR, *pErrorCode);
}
/* + basename */
pathBuffer.append(packageStub.data()+1, packageStub.length()-1, *pErrorCode);
if (!suffix.empty()) /* tack on suffix */
{
if (suffix.length() > 4) {
// If the suffix is actually an item ("ibm-5348_P100-1997.cnv") and not an extension (".res")
// then we need to ensure that the path ends with a separator.
pathBuffer.ensureEndsWithFileSeparator(*pErrorCode);
}
pathBuffer.append(suffix, *pErrorCode);
}
}
#ifdef UDATA_DEBUG
fprintf(stderr, " --> %s\n", pathBuffer.data());
#endif
return pathBuffer.data();
} while(path);
/* fell way off the end */
return NULL;
}
U_NAMESPACE_END
/* ==================================================================================*/
/*----------------------------------------------------------------------*
* *
* Add a static reference to the common data library *
* Unless overridden by an explicit udata_setCommonData, this will be *
* our common data. *
* *
*----------------------------------------------------------------------*/
#if U_PLATFORM_HAS_WINUWP_API == 0 // Windows UWP Platform does not support dll icu data at this time
extern "C" const DataHeader U_DATA_API U_ICUDATA_ENTRY_POINT;
#endif
/*
* This would be a good place for weak-linkage declarations of
* partial-data-library access functions where each returns a pointer
* to its data package, if it is linked in.
*/
/*
extern const void *uprv_getICUData_collation(void) ATTRIBUTE_WEAK;
extern const void *uprv_getICUData_conversion(void) ATTRIBUTE_WEAK;
*/
/*----------------------------------------------------------------------*
* *
* openCommonData Attempt to open a common format (.dat) file *
* Map it into memory (if it's not there already) *
* and return a UDataMemory object for it. *
* *
* If the requested data is already open and cached *
* just return the cached UDataMem object. *
* *
*----------------------------------------------------------------------*/
static UDataMemory *
openCommonData(const char *path, /* Path from OpenChoice? */
int32_t commonDataIndex, /* ICU Data (index >= 0) if path == NULL */
UErrorCode *pErrorCode)
{
UDataMemory tData;
const char *pathBuffer;
const char *inBasename;
if (U_FAILURE(*pErrorCode)) {
return NULL;
}
UDataMemory_init(&tData);
/* ??????? TODO revisit this */
if (commonDataIndex >= 0) {
/* "mini-cache" for common ICU data */
if(commonDataIndex >= UPRV_LENGTHOF(gCommonICUDataArray)) {
return NULL;
}
{
Mutex lock;
if(gCommonICUDataArray[commonDataIndex] != NULL) {
return gCommonICUDataArray[commonDataIndex];
}
#if U_PLATFORM_HAS_WINUWP_API == 0 // Windows UWP Platform does not support dll icu data at this time
int32_t i;
for(i = 0; i < commonDataIndex; ++i) {
if(gCommonICUDataArray[i]->pHeader == &U_ICUDATA_ENTRY_POINT) {
/* The linked-in data is already in the list. */
return NULL;
}
}
#endif
}
/* Add the linked-in data to the list. */
/*
* This is where we would check and call weakly linked partial-data-library
* access functions.
*/
/*
if (uprv_getICUData_collation) {
setCommonICUDataPointer(uprv_getICUData_collation(), FALSE, pErrorCode);
}
if (uprv_getICUData_conversion) {
setCommonICUDataPointer(uprv_getICUData_conversion(), FALSE, pErrorCode);
}
*/
#if U_PLATFORM_HAS_WINUWP_API == 0 // Windows UWP Platform does not support dll icu data at this time
setCommonICUDataPointer(&U_ICUDATA_ENTRY_POINT, FALSE, pErrorCode);
{
Mutex lock;
return gCommonICUDataArray[commonDataIndex];
}
#endif
}
/* request is NOT for ICU Data. */
/* Find the base name portion of the supplied path. */
/* inBasename will be left pointing somewhere within the original path string. */
inBasename = findBasename(path);
#ifdef UDATA_DEBUG
fprintf(stderr, "inBasename = %s\n", inBasename);
#endif
if(*inBasename==0) {
/* no basename. This will happen if the original path was a directory name, */
/* like "a/b/c/". (Fallback to separate files will still work.) */
#ifdef UDATA_DEBUG
fprintf(stderr, "ocd: no basename in %s, bailing.\n", path);
#endif
if (U_SUCCESS(*pErrorCode)) {
*pErrorCode=U_FILE_ACCESS_ERROR;
}
return NULL;
}
/* Is the requested common data file already open and cached? */
/* Note that the cache is keyed by the base name only. The rest of the path, */
/* if any, is not considered. */
UDataMemory *dataToReturn = udata_findCachedData(inBasename, *pErrorCode);
if (dataToReturn != NULL || U_FAILURE(*pErrorCode)) {
return dataToReturn;
}
/* Requested item is not in the cache.
* Hunt it down, trying all the path locations
*/
UDataPathIterator iter(u_getDataDirectory(), inBasename, path, ".dat", TRUE, pErrorCode);
while ((UDataMemory_isLoaded(&tData)==FALSE) && (pathBuffer = iter.next(pErrorCode)) != NULL)
{
#ifdef UDATA_DEBUG
fprintf(stderr, "ocd: trying path %s - ", pathBuffer);
#endif
uprv_mapFile(&tData, pathBuffer, pErrorCode);
#ifdef UDATA_DEBUG
fprintf(stderr, "%s\n", UDataMemory_isLoaded(&tData)?"LOADED":"not loaded");
#endif
}
if (U_FAILURE(*pErrorCode)) {
return NULL;
}
#if defined(OS390_STUBDATA) && defined(OS390BATCH)
if (!UDataMemory_isLoaded(&tData)) {
char ourPathBuffer[1024];
/* One more chance, for extendCommonData() */
uprv_strncpy(ourPathBuffer, path, 1019);
ourPathBuffer[1019]=0;
uprv_strcat(ourPathBuffer, ".dat");
uprv_mapFile(&tData, ourPathBuffer, pErrorCode);
}
#endif
if (U_FAILURE(*pErrorCode)) {
return NULL;
}
if (!UDataMemory_isLoaded(&tData)) {
/* no common data */
*pErrorCode=U_FILE_ACCESS_ERROR;
return NULL;
}
/* we have mapped a file, check its header */
udata_checkCommonData(&tData, pErrorCode);
/* Cache the UDataMemory struct for this .dat file,
* so we won't need to hunt it down and map it again next time
* something is needed from it. */
return udata_cacheDataItem(inBasename, &tData, pErrorCode);
}
/*----------------------------------------------------------------------*
* *
* extendICUData If the full set of ICU data was not loaded at *
* program startup, load it now. This function will *
* be called when the lookup of an ICU data item in *
* the common ICU data fails. *
* *
* return true if new data is loaded, false otherwise.*
* *
*----------------------------------------------------------------------*/
static UBool extendICUData(UErrorCode *pErr)
{
UDataMemory *pData;
UDataMemory copyPData;
UBool didUpdate = FALSE;
/*
* There is a chance for a race condition here.
* Normally, ICU data is loaded from a DLL or via mmap() and
* setCommonICUData() will detect if the same address is set twice.
* If ICU is built with data loading via fread() then the address will
* be different each time the common data is loaded and we may add
* multiple copies of the data.
* In this case, use a mutex to prevent the race.
* Use a specific mutex to avoid nested locks of the global mutex.
*/
#if MAP_IMPLEMENTATION==MAP_STDIO
static UMutex extendICUDataMutex = U_MUTEX_INITIALIZER;
umtx_lock(&extendICUDataMutex);
#endif
if(!umtx_loadAcquire(gHaveTriedToLoadCommonData)) {
/* See if we can explicitly open a .dat file for the ICUData. */
pData = openCommonData(
U_ICUDATA_NAME, /* "icudt20l" , for example. */
-1, /* Pretend we're not opening ICUData */
pErr);
/* How about if there is no pData, eh... */
UDataMemory_init(©PData);
if(pData != NULL) {
UDatamemory_assign(©PData, pData);
copyPData.map = 0; /* The mapping for this data is owned by the hash table */
copyPData.mapAddr = 0; /* which will unmap it when ICU is shut down. */
/* CommonICUData is also unmapped when ICU is shut down.*/
/* To avoid unmapping the data twice, zero out the map */
/* fields in the UDataMemory that we're assigning */
/* to CommonICUData. */
didUpdate = /* no longer using this result */
setCommonICUData(©PData,/* The new common data. */
FALSE, /* No warnings if write didn't happen */
pErr); /* setCommonICUData honors errors; NOP if error set */
}
umtx_storeRelease(gHaveTriedToLoadCommonData, 1);
}
didUpdate = findCommonICUDataByName(U_ICUDATA_NAME, *pErr); /* Return 'true' when a racing writes out the extended */
/* data after another thread has failed to see it (in openCommonData), so */
/* extended data can be examined. */
/* Also handles a race through here before gHaveTriedToLoadCommonData is set. */
#if MAP_IMPLEMENTATION==MAP_STDIO
umtx_unlock(&extendICUDataMutex);
#endif
return didUpdate; /* Return true if ICUData pointer was updated. */
/* (Could potentially have been done by another thread racing */
/* us through here, but that's fine, we still return true */
/* so that current thread will also examine extended data. */
}
/*----------------------------------------------------------------------*
* *
* udata_setCommonData *
* *
*----------------------------------------------------------------------*/
U_CAPI void U_EXPORT2
udata_setCommonData(const void *data, UErrorCode *pErrorCode) {
UDataMemory dataMemory;
if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
return;
}
if(data==NULL) {
*pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
return;
}
/* set the data pointer and test for validity */
UDataMemory_init(&dataMemory);
UDataMemory_setData(&dataMemory, data);
udata_checkCommonData(&dataMemory, pErrorCode);
if (U_FAILURE(*pErrorCode)) {return;}
/* we have good data */
/* Set it up as the ICU Common Data. */
setCommonICUData(&dataMemory, TRUE, pErrorCode);
}
/*---------------------------------------------------------------------------
*
* udata_setAppData
*
*---------------------------------------------------------------------------- */
U_CAPI void U_EXPORT2
udata_setAppData(const char *path, const void *data, UErrorCode *err)
{
UDataMemory udm;
if(err==NULL || U_FAILURE(*err)) {
return;
}
if(data==NULL) {
*err=U_ILLEGAL_ARGUMENT_ERROR;
return;
}
UDataMemory_init(&udm);
UDataMemory_setData(&udm, data);
udata_checkCommonData(&udm, err);
udata_cacheDataItem(path, &udm, err);
}
/*----------------------------------------------------------------------------*
* *
* checkDataItem Given a freshly located/loaded data item, either *
* an entry in a common file or a separately loaded file, *
* sanity check its header, and see if the data is *
* acceptable to the app. *
* If the data is good, create and return a UDataMemory *
* object that can be returned to the application. *
* Return NULL on any sort of failure. *
* *
*----------------------------------------------------------------------------*/
static UDataMemory *
checkDataItem
(
const DataHeader *pHeader, /* The data item to be checked. */
UDataMemoryIsAcceptable *isAcceptable, /* App's call-back function */
void *context, /* pass-thru param for above. */
const char *type, /* pass-thru param for above. */
const char *name, /* pass-thru param for above. */
UErrorCode *nonFatalErr, /* Error code if this data was not acceptable */
/* but openChoice should continue with */
/* trying to get data from fallback path. */
UErrorCode *fatalErr /* Bad error, caller should return immediately */
)
{
UDataMemory *rDataMem = NULL; /* the new UDataMemory, to be returned. */
if (U_FAILURE(*fatalErr)) {
return NULL;
}
if(pHeader->dataHeader.magic1==0xda &&
pHeader->dataHeader.magic2==0x27 &&
(isAcceptable==NULL || isAcceptable(context, type, name, &pHeader->info))
) {
rDataMem=UDataMemory_createNewInstance(fatalErr);
if (U_FAILURE(*fatalErr)) {
return NULL;
}
rDataMem->pHeader = pHeader;
} else {
/* the data is not acceptable, look further */
/* If we eventually find something good, this errorcode will be */
/* cleared out. */
*nonFatalErr=U_INVALID_FORMAT_ERROR;
}
return rDataMem;
}
/**
* @return 0 if not loaded, 1 if loaded or err
*/
static UDataMemory *doLoadFromIndividualFiles(const char *pkgName,
const char *dataPath, const char *tocEntryPathSuffix,
/* following arguments are the same as doOpenChoice itself */
const char *path, const char *type, const char *name,
UDataMemoryIsAcceptable *isAcceptable, void *context,
UErrorCode *subErrorCode,
UErrorCode *pErrorCode)
{
const char *pathBuffer;
UDataMemory dataMemory;
UDataMemory *pEntryData;
/* look in ind. files: package\nam.typ ========================= */
/* init path iterator for individual files */
UDataPathIterator iter(dataPath, pkgName, path, tocEntryPathSuffix, FALSE, pErrorCode);
while ((pathBuffer = iter.next(pErrorCode)) != NULL)
{
#ifdef UDATA_DEBUG
fprintf(stderr, "UDATA: trying individual file %s\n", pathBuffer);
#endif
if (uprv_mapFile(&dataMemory, pathBuffer, pErrorCode))
{
pEntryData = checkDataItem(dataMemory.pHeader, isAcceptable, context, type, name, subErrorCode, pErrorCode);
if (pEntryData != NULL) {
/* Data is good.
* Hand off ownership of the backing memory to the user's UDataMemory.
* and return it. */
pEntryData->mapAddr = dataMemory.mapAddr;
pEntryData->map = dataMemory.map;
#ifdef UDATA_DEBUG
fprintf(stderr, "** Mapped file: %s\n", pathBuffer);
#endif
return pEntryData;
}
/* the data is not acceptable, or some error occurred. Either way, unmap the memory */
udata_close(&dataMemory);
/* If we had a nasty error, bail out completely. */
if (U_FAILURE(*pErrorCode)) {
return NULL;
}
/* Otherwise remember that we found data but didn't like it for some reason */
*subErrorCode=U_INVALID_FORMAT_ERROR;
}
#ifdef UDATA_DEBUG
fprintf(stderr, "%s\n", UDataMemory_isLoaded(&dataMemory)?"LOADED":"not loaded");
#endif
}
return NULL;
}
/**
* @return 0 if not loaded, 1 if loaded or err
*/
static UDataMemory *doLoadFromCommonData(UBool isICUData, const char * /*pkgName*/,
const char * /*dataPath*/, const char * /*tocEntryPathSuffix*/, const char *tocEntryName,
/* following arguments are the same as doOpenChoice itself */
const char *path, const char *type, const char *name,
UDataMemoryIsAcceptable *isAcceptable, void *context,
UErrorCode *subErrorCode,
UErrorCode *pErrorCode)
{
UDataMemory *pEntryData;
const DataHeader *pHeader;
UDataMemory *pCommonData;
int32_t commonDataIndex;
UBool checkedExtendedICUData = FALSE;
/* try to get common data. The loop is for platforms such as the 390 that do
* not initially load the full set of ICU data. If the lookup of an ICU data item
* fails, the full (but slower to load) set is loaded, the and the loop repeats,
* trying the lookup again. Once the full set of ICU data is loaded, the loop wont
* repeat because the full set will be checked the first time through.
*
* The loop also handles the fallback to a .dat file if the application linked
* to the stub data library rather than a real library.
*/
for (commonDataIndex = isICUData ? 0 : -1;;) {
pCommonData=openCommonData(path, commonDataIndex, subErrorCode); /** search for pkg **/
if(U_SUCCESS(*subErrorCode) && pCommonData!=NULL) {
int32_t length;
/* look up the data piece in the common data */
pHeader=pCommonData->vFuncs->Lookup(pCommonData, tocEntryName, &length, subErrorCode);
#ifdef UDATA_DEBUG
fprintf(stderr, "%s: pHeader=%p - %s\n", tocEntryName, pHeader, u_errorName(*subErrorCode));
#endif
if(pHeader!=NULL) {
pEntryData = checkDataItem(pHeader, isAcceptable, context, type, name, subErrorCode, pErrorCode);
#ifdef UDATA_DEBUG
fprintf(stderr, "pEntryData=%p\n", pEntryData);
#endif
if (U_FAILURE(*pErrorCode)) {
return NULL;
}
if (pEntryData != NULL) {
pEntryData->length = length;
return pEntryData;
}
}
}
// If we failed due to being out-of-memory, then stop early and report the error.
if (*subErrorCode == U_MEMORY_ALLOCATION_ERROR) {
*pErrorCode = *subErrorCode;
return NULL;
}
/* Data wasn't found. If we were looking for an ICUData item and there is
* more data available, load it and try again,
* otherwise break out of this loop. */
if (!isICUData) {
return NULL;
} else if (pCommonData != NULL) {
++commonDataIndex; /* try the next data package */
} else if ((!checkedExtendedICUData) && extendICUData(subErrorCode)) {
checkedExtendedICUData = TRUE;
/* try this data package slot again: it changed from NULL to non-NULL */
} else {
return NULL;
}
}
}
/*
* Identify the Time Zone resources that are subject to special override data loading.
*/
static UBool isTimeZoneFile(const char *name, const char *type) {
return ((uprv_strcmp(type, "res") == 0) &&
(uprv_strcmp(name, "zoneinfo64") == 0 ||
uprv_strcmp(name, "timezoneTypes") == 0 ||
uprv_strcmp(name, "windowsZones") == 0 ||
uprv_strcmp(name, "metaZones") == 0));
}
/*
* A note on the ownership of Mapped Memory
*
* For common format files, ownership resides with the UDataMemory object
* that lives in the cache of opened common data. These UDataMemorys are private
* to the udata implementation, and are never seen directly by users.
*
* The UDataMemory objects returned to users will have the address of some desired
* data within the mapped region, but they wont have the mapping info itself, and thus
* won't cause anything to be removed from memory when they are closed.
*
* For individual data files, the UDataMemory returned to the user holds the
* information necessary to unmap the data on close. If the user independently
* opens the same data file twice, two completely independent mappings will be made.
* (There is no cache of opened data items from individual files, only a cache of
* opened Common Data files, that is, files containing a collection of data items.)
*
* For common data passed in from the user via udata_setAppData() or
* udata_setCommonData(), ownership remains with the user.
*
* UDataMemory objects themselves, as opposed to the memory they describe,
* can be anywhere - heap, stack/local or global.
* They have a flag to indicate when they're heap allocated and thus
* must be deleted when closed.
*/
/*----------------------------------------------------------------------------*
* *
* main data loading functions *
* *
*----------------------------------------------------------------------------*/
static UDataMemory *
doOpenChoice(const char *path, const char *type, const char *name,
UDataMemoryIsAcceptable *isAcceptable, void *context,
UErrorCode *pErrorCode)
{
UDataMemory *retVal = NULL;
const char *dataPath;
int32_t tocEntrySuffixIndex;
const char *tocEntryPathSuffix;
UErrorCode subErrorCode=U_ZERO_ERROR;
const char *treeChar;
UBool isICUData = FALSE;
/* Is this path ICU data? */
if(path == NULL ||
!strcmp(path, U_ICUDATA_ALIAS) || /* "ICUDATA" */
!uprv_strncmp(path, U_ICUDATA_NAME U_TREE_SEPARATOR_STRING, /* "icudt26e-" */
uprv_strlen(U_ICUDATA_NAME U_TREE_SEPARATOR_STRING)) ||
!uprv_strncmp(path, U_ICUDATA_ALIAS U_TREE_SEPARATOR_STRING, /* "ICUDATA-" */
uprv_strlen(U_ICUDATA_ALIAS U_TREE_SEPARATOR_STRING))) {
isICUData = TRUE;
}
#if (U_FILE_SEP_CHAR != U_FILE_ALT_SEP_CHAR) /* Windows: try "foo\bar" and "foo/bar" */
/* remap from alternate path char to the main one */
CharString altSepPath;
if(path) {
if(uprv_strchr(path,U_FILE_ALT_SEP_CHAR) != NULL) {
altSepPath.append(path, *pErrorCode);
char *p;
while ((p = uprv_strchr(altSepPath.data(), U_FILE_ALT_SEP_CHAR)) != NULL) {
*p = U_FILE_SEP_CHAR;
}
#if defined (UDATA_DEBUG)
fprintf(stderr, "Changed path from [%s] to [%s]\n", path, altSepPath.s);
#endif
path = altSepPath.data();
}
}
#endif
CharString tocEntryName; /* entry name in tree format. ex: 'icudt28b/coll/ar.res' */
CharString tocEntryPath; /* entry name in path format. ex: 'icudt28b\\coll\\ar.res' */
CharString pkgName;
CharString treeName;
/* ======= Set up strings */
if(path==NULL) {
pkgName.append(U_ICUDATA_NAME, *pErrorCode);
} else {
const char *pkg;
const char *first;
pkg = uprv_strrchr(path, U_FILE_SEP_CHAR);
first = uprv_strchr(path, U_FILE_SEP_CHAR);
if(uprv_pathIsAbsolute(path) || (pkg != first)) { /* more than one slash in the path- not a tree name */
/* see if this is an /absolute/path/to/package path */
if(pkg) {
pkgName.append(pkg+1, *pErrorCode);
} else {
pkgName.append(path, *pErrorCode);
}
} else {
treeChar = uprv_strchr(path, U_TREE_SEPARATOR);
if(treeChar) {
treeName.append(treeChar+1, *pErrorCode); /* following '-' */
if(isICUData) {
pkgName.append(U_ICUDATA_NAME, *pErrorCode);
} else {
pkgName.append(path, (int32_t)(treeChar-path), *pErrorCode);
if (first == NULL) {
/*
This user data has no path, but there is a tree name.
Look up the correct path from the data cache later.
*/
path = pkgName.data();
}
}
} else {
if(isICUData) {
pkgName.append(U_ICUDATA_NAME, *pErrorCode);
} else {
pkgName.append(path, *pErrorCode);
}
}
}
}
#ifdef UDATA_DEBUG
fprintf(stderr, " P=%s T=%s\n", pkgName.data(), treeName.data());
#endif
/* setting up the entry name and file name
* Make up a full name by appending the type to the supplied
* name, assuming that a type was supplied.
*/
/* prepend the package */
tocEntryName.append(pkgName, *pErrorCode);
tocEntryPath.append(pkgName, *pErrorCode);
tocEntrySuffixIndex = tocEntryName.length();
if(!treeName.isEmpty()) {
tocEntryName.append(U_TREE_ENTRY_SEP_CHAR, *pErrorCode).append(treeName, *pErrorCode);
tocEntryPath.append(U_FILE_SEP_CHAR, *pErrorCode).append(treeName, *pErrorCode);
}
tocEntryName.append(U_TREE_ENTRY_SEP_CHAR, *pErrorCode).append(name, *pErrorCode);
tocEntryPath.append(U_FILE_SEP_CHAR, *pErrorCode).append(name, *pErrorCode);
if(type!=NULL && *type!=0) {
tocEntryName.append(".", *pErrorCode).append(type, *pErrorCode);
tocEntryPath.append(".", *pErrorCode).append(type, *pErrorCode);
}
// The +1 is for the U_FILE_SEP_CHAR that is always appended above.
tocEntryPathSuffix = tocEntryPath.data() + tocEntrySuffixIndex + 1; /* suffix starts here */
#ifdef UDATA_DEBUG
fprintf(stderr, " tocEntryName = %s\n", tocEntryName.data());
fprintf(stderr, " tocEntryPath = %s\n", tocEntryName.data());
#endif
#if U_PLATFORM_HAS_WINUWP_API == 0 // Windows UWP Platform does not support dll icu data at this time
if(path == NULL) {
path = COMMON_DATA_NAME; /* "icudt26e" */
}
#else
// Windows UWP expects only a single data file.
path = COMMON_DATA_NAME; /* "icudt26e" */
#endif
/************************ Begin loop looking for ind. files ***************/
#ifdef UDATA_DEBUG
fprintf(stderr, "IND: inBasename = %s, pkg=%s\n", "(n/a)", packageNameFromPath(path));
#endif
/* End of dealing with a null basename */
dataPath = u_getDataDirectory();
/**** Time zone individual files override */
if (isICUData && isTimeZoneFile(name, type)) {
const char *tzFilesDir = u_getTimeZoneFilesDirectory(pErrorCode);
if (tzFilesDir[0] != 0) {
#ifdef UDATA_DEBUG
fprintf(stderr, "Trying Time Zone Files directory = %s\n", tzFilesDir);
#endif
retVal = doLoadFromIndividualFiles(/* pkgName.data() */ "", tzFilesDir, tocEntryPathSuffix,
/* path */ "", type, name, isAcceptable, context, &subErrorCode, pErrorCode);
if((retVal != NULL) || U_FAILURE(*pErrorCode)) {
return retVal;
}
}
}
/**** COMMON PACKAGE - only if packages are first. */
if(gDataFileAccess == UDATA_PACKAGES_FIRST) {
#ifdef UDATA_DEBUG
fprintf(stderr, "Trying packages (UDATA_PACKAGES_FIRST)\n");
#endif
/* #2 */
retVal = doLoadFromCommonData(isICUData,
pkgName.data(), dataPath, tocEntryPathSuffix, tocEntryName.data(),
path, type, name, isAcceptable, context, &subErrorCode, pErrorCode);
if((retVal != NULL) || U_FAILURE(*pErrorCode)) {
return retVal;
}
}
/**** INDIVIDUAL FILES */
if((gDataFileAccess==UDATA_PACKAGES_FIRST) ||
(gDataFileAccess==UDATA_FILES_FIRST)) {
#ifdef UDATA_DEBUG
fprintf(stderr, "Trying individual files\n");
#endif
/* Check to make sure that there is a dataPath to iterate over */
if ((dataPath && *dataPath) || !isICUData) {
retVal = doLoadFromIndividualFiles(pkgName.data(), dataPath, tocEntryPathSuffix,
path, type, name, isAcceptable, context, &subErrorCode, pErrorCode);
if((retVal != NULL) || U_FAILURE(*pErrorCode)) {
return retVal;
}
}
}
/**** COMMON PACKAGE */
if((gDataFileAccess==UDATA_ONLY_PACKAGES) ||
(gDataFileAccess==UDATA_FILES_FIRST)) {
#ifdef UDATA_DEBUG
fprintf(stderr, "Trying packages (UDATA_ONLY_PACKAGES || UDATA_FILES_FIRST)\n");
#endif
retVal = doLoadFromCommonData(isICUData,
pkgName.data(), dataPath, tocEntryPathSuffix, tocEntryName.data(),
path, type, name, isAcceptable, context, &subErrorCode, pErrorCode);
if((retVal != NULL) || U_FAILURE(*pErrorCode)) {
return retVal;
}
}
/* Load from DLL. If we haven't attempted package load, we also haven't had any chance to
try a DLL (static or setCommonData/etc) load.
If we ever have a "UDATA_ONLY_FILES", add it to the or list here. */
if(gDataFileAccess==UDATA_NO_FILES) {
#ifdef UDATA_DEBUG
fprintf(stderr, "Trying common data (UDATA_NO_FILES)\n");
#endif
retVal = doLoadFromCommonData(isICUData,
pkgName.data(), "", tocEntryPathSuffix, tocEntryName.data(),
path, type, name, isAcceptable, context, &subErrorCode, pErrorCode);
if((retVal != NULL) || U_FAILURE(*pErrorCode)) {
return retVal;
}
}
/* data not found */
if(U_SUCCESS(*pErrorCode)) {
if(U_SUCCESS(subErrorCode)) {
/* file not found */
*pErrorCode=U_FILE_ACCESS_ERROR;
} else {
/* entry point not found or rejected */
*pErrorCode=subErrorCode;
}
}
return retVal;
}
/* API ---------------------------------------------------------------------- */
U_CAPI UDataMemory * U_EXPORT2
udata_open(const char *path, const char *type, const char *name,
UErrorCode *pErrorCode) {
#ifdef UDATA_DEBUG
fprintf(stderr, "udata_open(): Opening: %s : %s . %s\n", (path?path:"NULL"), name, type);
fflush(stderr);
#endif
if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
return NULL;
} else if(name==NULL || *name==0) {
*pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
return NULL;
} else {
return doOpenChoice(path, type, name, NULL, NULL, pErrorCode);
}
}
U_CAPI UDataMemory * U_EXPORT2
udata_openChoice(const char *path, const char *type, const char *name,
UDataMemoryIsAcceptable *isAcceptable, void *context,
UErrorCode *pErrorCode) {
#ifdef UDATA_DEBUG
fprintf(stderr, "udata_openChoice(): Opening: %s : %s . %s\n", (path?path:"NULL"), name, type);
#endif
if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
return NULL;
} else if(name==NULL || *name==0 || isAcceptable==NULL) {
*pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
return NULL;
} else {
return doOpenChoice(path, type, name, isAcceptable, context, pErrorCode);
}
}
U_CAPI void U_EXPORT2
udata_getInfo(UDataMemory *pData, UDataInfo *pInfo) {
if(pInfo!=NULL) {
if(pData!=NULL && pData->pHeader!=NULL) {
const UDataInfo *info=&pData->pHeader->info;
uint16_t dataInfoSize=udata_getInfoSize(info);
if(pInfo->size>dataInfoSize) {
pInfo->size=dataInfoSize;
}
uprv_memcpy((uint16_t *)pInfo+1, (const uint16_t *)info+1, pInfo->size-2);
if(info->isBigEndian!=U_IS_BIG_ENDIAN) {
/* opposite endianness */
uint16_t x=info->reservedWord;
pInfo->reservedWord=(uint16_t)((x<<8)|(x>>8));
}
} else {
pInfo->size=0;
}
}
}
U_CAPI void U_EXPORT2 udata_setFileAccess(UDataFileAccess access, UErrorCode * /*status*/)
{
// Note: this function is documented as not thread safe.
gDataFileAccess = access;
}