vocabulary.h

/* ScummVM - Graphic Adventure Engine
 *
 * ScummVM is the legal property of its developers, whose names
 * are too numerous to list here. Please refer to the COPYRIGHT
 * file distributed with this source distribution.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

#ifndef SCI_PARSER_VOCABULARY_H
#define SCI_PARSER_VOCABULARY_H

#include "common/str.h"
#include "common/hashmap.h"
#include "common/hash-str.h"
#include "common/list.h"

#include "sci/sci.h"
#include "sci/engine/vm_types.h"
#include "sci/util.h"

namespace Common {

class Serializer;

}

namespace Sci {

class ResourceManager;

/*#define VOCABULARY_DEBUG */

enum {
    VOCAB_RESOURCE_SELECTORS = 997,

    VOCAB_RESOURCE_SCI0_MAIN_VOCAB = 0,
    VOCAB_RESOURCE_SCUMM_LOC_VOCAB = 1, // Special fanmade format for vocab translate
    VOCAB_RESOURCE_SCI0_PARSE_TREE_BRANCHES = 900,
    VOCAB_RESOURCE_SCI0_SUFFIX_VOCAB = 901,

    VOCAB_RESOURCE_SCI1_MAIN_VOCAB = 900,
    VOCAB_RESOURCE_SCI1_PARSE_TREE_BRANCHES = 901,
    VOCAB_RESOURCE_SCI1_SUFFIX_VOCAB = 902,

    VOCAB_RESOURCE_ALT_INPUTS = 913
};


enum {
    VOCAB_CLASS_PREPOSITION = 0x01,
    VOCAB_CLASS_ARTICLE = 0x02,
    VOCAB_CLASS_ADJECTIVE = 0x04,
    VOCAB_CLASS_PRONOUN = 0x08,
    VOCAB_CLASS_NOUN = 0x10,
    VOCAB_CLASS_INDICATIVE_VERB = 0x20,
    VOCAB_CLASS_ADVERB = 0x40,
    VOCAB_CLASS_IMPERATIVE_VERB = 0x80,
    VOCAB_CLASS_NUMBER = 0x001
};

enum {
    kParseEndOfInput = 0,
    kParseOpeningParenthesis = 1,
    kParseClosingParenthesis = 2,
    kParseNil = 3,
    kParseNumber = 4
};

#define VOCAB_MAX_WORDLENGTH 256

/* Anywords are ignored by the parser */
#define VOCAB_CLASS_ANYWORD 0xff

/* This word class is used for numbers */
#define VOCAB_MAGIC_NUMBER_GROUP 0xffd /* 0xffe ? */
#define VOCAB_MAGIC_NOTHING_GROUP 0xffe

/* Number of nodes for each ParseTreeNode structure */
#define VOCAB_TREE_NODES 500

#define VOCAB_TREE_NODE_LAST_WORD_STORAGE 0x140
#define VOCAB_TREE_NODE_COMPARE_TYPE 0x146
#define VOCAB_TREE_NODE_COMPARE_GROUP 0x14d
#define VOCAB_TREE_NODE_FORCE_STORAGE 0x154

#define SAID_COMMA   0xf0
#define SAID_AMP     0xf1
#define SAID_SLASH   0xf2
#define SAID_PARENO  0xf3
#define SAID_PARENC  0xf4
#define SAID_BRACKO  0xf5
#define SAID_BRACKC  0xf6
#define SAID_HASH    0xf7
#define SAID_LT      0xf8
#define SAID_GT      0xf9
#define SAID_TERM    0xff

#define SAID_FIRST SAID_COMMA

/* There was no 'last matching word': */
#define SAID_FULL_MATCH 0xffff
#define SAID_NO_MATCH 0xfffe
#define SAID_PARTIAL_MATCH 0xfffd

#define SAID_LONG(x) ((x) << 8)

struct ResultWord {
    int _class; 
    int _group; 
};

typedef Common::List<ResultWord> ResultWordList;
typedef Common::List<ResultWordList> ResultWordListList;

typedef Common::HashMap<Common::String, ResultWordList, Common::CaseSensitiveString_Hash, Common::CaseSensitiveString_EqualTo> WordMap;


struct ParseRuleList;

struct suffix_t {

    int class_mask; 
    int result_class; 
    int alt_suffix_length; 
    int word_suffix_length; 
    const char *alt_suffix; 
    const char *word_suffix; 
};

typedef Common::List<suffix_t> SuffixList;


struct synonym_t {
    uint16 replaceant; 
    uint16 replacement; 
};

typedef Common::Array<synonym_t> SynonymList;


struct AltInput {
    const char *_input;
    const char *_replacement;
    uint32 _inputLength;
    bool _prefix;
};


struct parse_tree_branch_t {
    int id;
    int data[10];
};

enum ParseTypes {
    kParseTreeWordNode = 4,
    kParseTreeLeafNode = 5,
    kParseTreeBranchNode = 6
};

struct ParseTreeNode {
    ParseTypes type;  
    int value; 
    ParseTreeNode* left; 
    ParseTreeNode* right; 
};

enum VocabularyVersions {
    kVocabularySCI0 = 0,
    kVocabularySCI1 = 1
};

class Vocabulary {
public:
    Vocabulary(ResourceManager *resMan, bool foreign);
    ~Vocabulary();

    // reset parser status
    void reset();

    const char *getAnyWordFromGroup(int group);


    void lookupWord(ResultWordList &retval, const char *word, int word_len);

    void lookupWordPrefix(ResultWordListList &parent_retval, ResultWordList &retval, const char *word, int word_len);

    bool lookupSpecificPrefixWithMeaning(ResultWordListList &parent_retval, ResultWordList &retval, const char *word, int word_len, unsigned char prefix, const char *meaning);

    bool lookupVerbPrefix(ResultWordListList &parent_retval, ResultWordList &retval, Common::String word, int word_len, Common::String prefix);

    bool tokenizeString(ResultWordListList &retval, const char *sentence, char **error);

    int parseGNF(const ResultWordListList &words, bool verbose = false);

    bool storePronounReference();

    void replacePronouns(ResultWordListList &words);

    ParseRuleList *buildGNF(bool verbose = false);

    void debugDecipherSaidBlock(const SciSpan<const byte> &data);

    void printSuffixes() const;

    void printParserWords() const;

    uint getParserBranchesSize() const { return _parserBranches.size(); }
    const parse_tree_branch_t &getParseTreeBranch(int number) const { return _parserBranches[number]; }

    void addSynonym(synonym_t syn) { _synonyms.push_back(syn); }

    void clearSynonyms() { _synonyms.clear(); }

    void synonymizeTokens(ResultWordListList &words);

    void printParserNodes(int num);

    void dumpParseTree();

    int parseNodes(int *i, int *pos, int type, int nr, int argc, const char **argv);

    bool checkAltInput(Common::String& text, uint16& cursorPos);

    void saveLoadWithSerializer(Common::Serializer &ser);

private:
    bool loadParserWords();
    
    void loadTranslatedWords();

    bool loadSuffixes();

    void freeSuffixes();

    bool loadBranches();

    void freeRuleList(ParseRuleList *rule_list);


    bool loadAltInputs();

    void freeAltInputs();

    ResourceManager *_resMan;
    VocabularyVersions _vocabVersion;

    bool _foreign;
    uint16 _resourceIdWords;
    uint16 _resourceIdSuffixes;
    uint16 _resourceIdBranches;

    // Parser-related lists
    SuffixList _parserSuffixes;
    ParseRuleList *_parserRules; 
    Common::Array<parse_tree_branch_t> _parserBranches;
    WordMap _parserWords;
    SynonymList _synonyms; 
    Common::Array<Common::List<AltInput> > _altInputs;

    struct PrefixMeaning {
        unsigned char prefix;
        const char *meaning;
    };

    int _pronounReference;

public:
    // Accessed by said()
    ParseTreeNode _parserNodes[VOCAB_TREE_NODES]; 
    // Parser data:
    reg_t parser_event; 
    bool parserIsValid; 
};

void vocab_dump_parse_tree(const char *tree_name, ParseTreeNode *nodes);



int said(const byte *spec, bool verbose);

} // End of namespace Sci

#endif // SCI_PARSER_VOCABULARY_H