/*
* Copyright (C) 2008-2009 SVOX AG, Baslerstr. 30, 8048 Zuerich, Switzerland
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @file picokfst.c
*
* FST knowledge loading and access
*
* Copyright (C) 2008-2009 SVOX AG, Baslerstr. 30, 8048 Zuerich, Switzerland
* All rights reserved.
*
* History:
* - 2009-04-20 -- initial version
*
*/
#include "picoos.h"
#include "picodbg.h"
#include "picoknow.h"
#include "picokfst.h"
#ifdef __cplusplus
extern "C" {
#endif
#if 0
}
#endif
#define FileHdrSize 4 /* size of FST file header */
/* ************************************************************/
/* function to create specialized kb, */
/* to be used by picorsrc only */
/* ************************************************************/
/** object : FSTKnowledgeBase
* shortcut : kfst
* derived from : picoknow_KnowledgeBase
*/
typedef struct kfst_subobj * kfst_SubObj;
typedef struct kfst_subobj{
picoos_uint8 * fstStream; /* the byte stream base address */
picoos_int32 hdrLen; /* length of file header */
picoos_int32 transductionMode; /* transduction mode to be used for FST */
picoos_int32 nrClasses; /* nr of pair/transition classes in FST; class is in [1..nrClasses] */
picoos_int32 nrStates; /* nr of states in FST; state is in [1..nrState] */
picoos_int32 termClass; /* pair class of terminator symbol pair; probably obsolete */
picoos_int32 alphaHashTabSize; /* size of pair alphabet hash table */
picoos_int32 alphaHashTabPos; /* absolute address of the start of the pair alphabet */
picoos_int32 transTabEntrySize; /* size in bytes of each transition table entry */
picoos_int32 transTabPos; /* absolute address of the start of the transition table */
picoos_int32 inEpsStateTabPos; /* absolute address of the start of the input epsilon transition table */
picoos_int32 accStateTabPos; /* absolute address of the table of accepting states */
} kfst_subobj_t;
/* ************************************************************/
/* primitives for reading from byte stream */
/* ************************************************************/
/* Converts 'nrBytes' bytes starting at position '*pos' in byte stream 'stream' into unsigned number 'num'.
'*pos' is modified to the position right after the number */
static void FixedBytesToUnsignedNum (picoos_uint8 * stream, picoos_uint8 nrBytes, picoos_uint32 * pos, picoos_uint32 * num)
{
picoos_int32 i;
(*num) = 0;
for (i = 0; i < nrBytes; i++) {
(*num) = ((*num) << 8) + (picoos_uint32)stream[*pos];
(*pos)++;
}
}
/* Converts 'nrBytes' bytes starting at position '*pos' in byte stream 'stream' into signed number 'num'.
'*pos' is modified to the position right after the number */
static void FixedBytesToSignedNum (picoos_uint8 * stream, picoos_uint8 nrBytes, picoos_uint32 * pos, picoos_int32 * num)
{
picoos_int32 i;
picoos_uint32 val;
val = 0;
for (i = 0; i < nrBytes; i++) {
val = (val << 8) + (picoos_uint32)stream[*pos];
(*pos)++;
}
if (val % 2 == 1) {
/* negative number */
(*num) = -((picoos_int32)((val - 1) / 2)) - 1;
} else {
/* positive number */
(*num) = val / 2;
}
}
/* Converts varying-sized sequence of bytes starting at position '*pos' in byte stream 'stream'
into (signed) number 'num'. '*pos' is modified to the position right after the number. */
static void BytesToNum (picoos_uint8 * stream, picoos_uint32 * pos, picoos_int32 * num)
{
picoos_uint32 val;
picoos_uint32 b;
val = 0;
b = (picoos_uint32)stream[*pos];
(*pos)++;
while (b < 128) {
val = (val << 7) + b;
b = (picoos_uint32)stream[*pos];
(*pos)++;
}
val = (val << 7) + (b - 128);
if (val % 2 == 1) {
/* negative number */
(*num) = -((picoos_int32)((val - 1) / 2)) - 1;
} else {
/* positive number */
(*num) = val / 2;
}
}
/* ************************************************************/
/* setting up FST from byte stream */
/* ************************************************************/
static pico_status_t kfstInitialize(register picoknow_KnowledgeBase this,
picoos_Common common)
{
picoos_uint32 curpos;
picoos_int32 offs;
kfst_subobj_t * kfst;
PICODBG_DEBUG(("kfstInitialize -- start\n"));
if (NULL == this || NULL == this->subObj) {
return picoos_emRaiseException(common->em, PICO_EXC_KB_MISSING, NULL,
NULL);
}
kfst = (kfst_subobj_t *) this->subObj;
/* +CT+ */
kfst->fstStream = this->base;
PICODBG_TRACE(("base: %d\n",this->base));
kfst->hdrLen = FileHdrSize;
curpos = kfst->hdrLen;
BytesToNum(kfst->fstStream,& curpos,& kfst->transductionMode);
BytesToNum(kfst->fstStream,& curpos,& kfst->nrClasses);
BytesToNum(kfst->fstStream,& curpos,& kfst->nrStates);
BytesToNum(kfst->fstStream,& curpos,& kfst->termClass);
BytesToNum(kfst->fstStream,& curpos,& kfst->alphaHashTabSize);
BytesToNum(kfst->fstStream,& curpos,& offs);
kfst->alphaHashTabPos = kfst->hdrLen + offs;
BytesToNum(kfst->fstStream,& curpos,& kfst->transTabEntrySize);
BytesToNum(kfst->fstStream,& curpos,& offs);
kfst->transTabPos = kfst->hdrLen + offs;
BytesToNum(kfst->fstStream,& curpos,& offs);
kfst->inEpsStateTabPos = kfst->hdrLen + offs;
BytesToNum(kfst->fstStream,& curpos,& offs);
kfst->accStateTabPos = kfst->hdrLen + offs;
/* -CT- */
return PICO_OK;
}
static pico_status_t kfstSubObjDeallocate(register picoknow_KnowledgeBase this,
picoos_MemoryManager mm)
{
if (NULL != this) {
picoos_deallocate(mm, (void *) &this->subObj);
}
return PICO_OK;
}
/* calculates a small number of data (e.g. addresses) from kb for fast access.
* This data is encapsulated in a picokfst_FST that can later be retrieved
* with picokfst_getFST. */
pico_status_t picokfst_specializeFSTKnowledgeBase(picoknow_KnowledgeBase this,
picoos_Common common)
{
pico_status_t status;
if (NULL == this) {
return picoos_emRaiseException(common->em, PICO_EXC_KB_MISSING, NULL, NULL);
}
if (0 < this->size) {
/* not a dummy kb */
this->subDeallocate = kfstSubObjDeallocate;
this->subObj = picoos_allocate(common->mm, sizeof(kfst_subobj_t));
if (NULL == this->subObj) {
return picoos_emRaiseException(common->em, PICO_EXC_OUT_OF_MEM, NULL, NULL);
}
status = kfstInitialize(this, common);
if (PICO_OK != status) {
picoos_deallocate(common->mm,(void **)&this->subObj);
}
}
return PICO_OK;
}
/* ************************************************************/
/* FST type and getFST function */
/* ************************************************************/
/* return kb FST for usage in PU */
picokfst_FST picokfst_getFST(picoknow_KnowledgeBase this)
{
if (NULL == this) {
return NULL;
} else {
return (picokfst_FST) this->subObj;
}
}
/* ************************************************************/
/* FST access methods */
/* ************************************************************/
/* see description in header file */
extern picoos_uint8 picokfst_kfstGetTransductionMode(picokfst_FST this)
{
kfst_SubObj fst = (kfst_SubObj) this;
if (fst != NULL) {
return fst->transductionMode;
} else {
return 0;
}
}
/* see description in header file */
extern void picokfst_kfstGetFSTSizes (picokfst_FST this, picoos_int32 *nrStates, picoos_int32 *nrClasses)
{
kfst_SubObj fst = (kfst_SubObj) this;
if (fst != NULL) {
*nrStates = fst->nrStates;
*nrClasses = fst->nrClasses;
} else {
*nrStates = 0;
*nrClasses = 0;
}
}
/* see description in header file */
extern void picokfst_kfstStartPairSearch (picokfst_FST this, picokfst_symid_t inSym,
picoos_bool * inSymFound, picoos_int32 * searchState)
{
picoos_uint32 pos;
picoos_int32 offs;
picoos_int32 h;
picoos_int32 inSymCellPos;
picoos_int32 inSymX;
picoos_int32 nextSameHashInSymOffs;
kfst_SubObj fst = (kfst_SubObj) this;
(*searchState) = -1;
(*inSymFound) = 0;
h = inSym % fst->alphaHashTabSize;
pos = fst->alphaHashTabPos + (h * 4);
FixedBytesToSignedNum(fst->fstStream,4,& pos,& offs);
if (offs > 0) {
inSymCellPos = fst->alphaHashTabPos + offs;
pos = inSymCellPos;
BytesToNum(fst->fstStream,& pos,& inSymX);
BytesToNum(fst->fstStream,& pos,& nextSameHashInSymOffs);
while ((inSymX != inSym) && (nextSameHashInSymOffs > 0)) {
inSymCellPos = inSymCellPos + nextSameHashInSymOffs;
pos = inSymCellPos;
BytesToNum(fst->fstStream,& pos,& inSymX);
BytesToNum(fst->fstStream,& pos,& nextSameHashInSymOffs);
}
if (inSymX == inSym) {
/* input symbol found; state is set to position after symbol cell */
(*searchState) = pos;
(*inSymFound) = 1;
}
}
}
/* see description in header file */
extern void picokfst_kfstGetNextPair (picokfst_FST this, picoos_int32 * searchState,
picoos_bool * pairFound,
picokfst_symid_t * outSym, picokfst_class_t * pairClass)
{
picoos_uint32 pos;
picoos_int32 val;
kfst_SubObj fst = (kfst_SubObj) this;
if ((*searchState) < 0) {
(*pairFound) = 0;
(*outSym) = PICOKFST_SYMID_ILLEG;
(*pairClass) = -1;
} else {
pos = (*searchState);
BytesToNum(fst->fstStream,& pos,& val);
*outSym = (picokfst_symid_t)val;
if ((*outSym) != PICOKFST_SYMID_ILLEG) {
BytesToNum(fst->fstStream,& pos,& val);
*pairClass = (picokfst_class_t)val;
(*pairFound) = 1;
(*searchState) = pos;
} else {
(*pairFound) = 0;
(*outSym) = PICOKFST_SYMID_ILLEG;
(*pairClass) = -1;
(*searchState) = -1;
}
}
}
/* see description in header file */
extern void picokfst_kfstGetTrans (picokfst_FST this, picokfst_state_t startState, picokfst_class_t transClass,
picokfst_state_t * endState)
{
picoos_uint32 pos;
picoos_int32 index;
picoos_uint32 endStateX;
kfst_SubObj fst = (kfst_SubObj) this;
if ((startState < 1) || (startState > fst->nrStates) || (transClass < 1) || (transClass > fst->nrClasses)) {
(*endState) = 0;
} else {
index = (startState - 1) * fst->nrClasses + transClass - 1;
pos = fst->transTabPos + (index * fst->transTabEntrySize);
FixedBytesToUnsignedNum(fst->fstStream,fst->transTabEntrySize,& pos,& endStateX);
(*endState) = endStateX;
}
}
/* see description in header file */
extern void picokfst_kfstStartInEpsTransSearch (picokfst_FST this, picokfst_state_t startState,
picoos_bool * inEpsTransFound, picoos_int32 * searchState)
{
picoos_int32 offs;
picoos_uint32 pos;
kfst_SubObj fst = (kfst_SubObj) this;
(*searchState) = -1;
(*inEpsTransFound) = 0;
if ((startState > 0) && (startState <= fst->nrStates)) {
pos = fst->inEpsStateTabPos + (startState - 1) * 4;
FixedBytesToSignedNum(fst->fstStream,4,& pos,& offs);
if (offs > 0) {
(*searchState) = fst->inEpsStateTabPos + offs;
(*inEpsTransFound) = 1;
}
}
}
/* see description in header file */
extern void picokfst_kfstGetNextInEpsTrans (picokfst_FST this, picoos_int32 * searchState,
picoos_bool * inEpsTransFound,
picokfst_symid_t * outSym, picokfst_state_t * endState)
{
picoos_uint32 pos;
picoos_int32 val;
kfst_SubObj fst = (kfst_SubObj) this;
if ((*searchState) < 0) {
(*inEpsTransFound) = 0;
(*outSym) = PICOKFST_SYMID_ILLEG;
(*endState) = 0;
} else {
pos = (*searchState);
BytesToNum(fst->fstStream,& pos,& val);
*outSym = (picokfst_symid_t)val;
if ((*outSym) != PICOKFST_SYMID_ILLEG) {
BytesToNum(fst->fstStream,& pos,& val);
*endState = (picokfst_state_t)val;
(*inEpsTransFound) = 1;
(*searchState) = pos;
} else {
(*inEpsTransFound) = 0;
(*outSym) = PICOKFST_SYMID_ILLEG;
(*endState) = 0;
(*searchState) = -1;
}
}
}
/* see description in header file */
extern picoos_bool picokfst_kfstIsAcceptingState (picokfst_FST this, picokfst_state_t state)
{
picoos_uint32 pos;
picoos_uint32 val;
kfst_SubObj fst = (kfst_SubObj) this;
if ((state > 0) && (state <= fst->nrStates)) {
pos = fst->accStateTabPos + (state - 1);
FixedBytesToUnsignedNum(fst->fstStream,1,& pos,& val);
return (val == 1);
} else {
return 0;
}
}
#ifdef __cplusplus
}
#endif
/* End picofst.c */