Backups/googlepinyinime-rev
1
0
Fork 0
mirror of https://github.com/fanlumaster/googlepinyinime-rev.git synced 2025-07-18 08:57:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

adapt to win32

Browse Source
This commit is contained in:
fanlumaster
2025-01-02 06:19:47 +08:00
parent 155d6eb410
commit 12abe936bf
13 changed files with 2864 additions and 2714 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
command/pinyinime_dictbuilder.cpp
View File

@ -1,55 +1,59 @@
/* /*
* Copyright (C) 2009 The Android Open Source Project * Copyright (C) 2009 The Android Open Source Project
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
* *
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* *
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
#include <assert.h> #include <assert.h>
#include <stdlib.h> #include <stdlib.h>
#include <stdio.h> #include <stdio.h>
#include <time.h> #include <time.h>
#include <unistd.h> #ifdef _WIN32
#include "../src/include/dicttrie.h" #include <io.h>
#else
using namespace ime_pinyin; #include <unistd.h>
#endif
/** #include "../src/include/dicttrie.h"
* Build binary dictionary model. Make sure that ___BUILD_MODEL___ is defined
* in dictdef.h. using namespace ime_pinyin;
*/
int main(int argc, char* argv[]) { /**
DictTrie* dict_trie = new DictTrie(); * Build binary dictionary model. Make sure that ___BUILD_MODEL___ is defined
bool success; * in dictdef.h.
if (argc >= 3) */
success = dict_trie->build_dict(argv[1], argv[2]); int main(int argc, char* argv[]) {
else DictTrie* dict_trie = new DictTrie();
success = dict_trie->build_dict("../data/rawdict_utf16_65105_freq.txt", "../data/valid_utf16.txt"); bool success;
if (argc >= 3)
if (success) { success = dict_trie->build_dict(argv[1], argv[2]);
printf("Build dictionary successfully.\n"); else
} else { success = dict_trie->build_dict("../data/rawdict_utf16_65105_freq.txt", "../data/valid_utf16.txt");
printf("Build dictionary unsuccessfully.\n");
return -1; if (success) {
} printf("Build dictionary successfully.\n");
} else {
success = dict_trie->save_dict("./dict/dict_pinyin.dat"); printf("Build dictionary unsuccessfully.\n");
return -1;
if (success) { }
printf("Save dictionary successfully.\n");
} else { success = dict_trie->save_dict("./dict/dict_pinyin.dat");
printf("Save dictionary unsuccessfully.\n");
return -1; if (success) {
} printf("Save dictionary successfully.\n");
} else {
return 0; printf("Save dictionary unsuccessfully.\n");
return -1;
}
return 0;
} }

46
command/lcompile.sh → command/scripts/lcompile.sh Executable file → Normal file
View File

@ -1,23 +1,23 @@
#!/bin/bash #!/bin/bash
currentDirectory=$(pwd) currentDirectory=$(pwd)
cmakeListsPath="${currentDirectory}/CMakeLists.txt" cmakeListsPath="${currentDirectory}/CMakeLists.txt"
if [ ! -f "$cmakeListsPath" ]; then if [ ! -f "$cmakeListsPath" ]; then
echo "No CMakeLists.txt in current directory, please check." echo "No CMakeLists.txt in current directory, please check."
exit 1 exit 1
fi fi
echo "Start generating and compiling..." echo "Start generating and compiling..."
buildFolderPath="./build" buildFolderPath="./build"
if [ ! -d "$buildFolderPath" ]; then if [ ! -d "$buildFolderPath" ]; then
mkdir -p "$buildFolderPath" mkdir -p "$buildFolderPath"
echo "build folder created." echo "build folder created."
fi fi
cmake -G "Unix Makefiles" -D CMAKE_CXX_COMPILER=/usr/bin/g++ -S . -B ./build/ cmake -G "Unix Makefiles" -D CMAKE_CXX_COMPILER=/usr/bin/g++ -S . -B ./build/
if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
cmake --build ./build/ --config DEBUG cmake --build ./build/ --config DEBUG
fi fi

90
llaunch.sh → command/scripts/llaunch.sh Executable file → Normal file
View File

@ -1,45 +1,45 @@
#!/bin/bash #!/bin/bash
currentDirectory=$(pwd) currentDirectory=$(pwd)
cmakeListsPath="${currentDirectory}/CMakeLists.txt" cmakeListsPath="${currentDirectory}/CMakeLists.txt"
if [ ! -f "$cmakeListsPath" ]; then if [ ! -f "$cmakeListsPath" ]; then
echo "No CMakeLists.txt in current directory, please check." echo "No CMakeLists.txt in current directory, please check."
exit 1 exit 1
fi fi
echo "Start generating and compiling..." echo "Start generating and compiling..."
buildFolderPath="./build" buildFolderPath="./build"
if [ ! -d "$buildFolderPath" ]; then if [ ! -d "$buildFolderPath" ]; then
mkdir -p "$buildFolderPath" mkdir -p "$buildFolderPath"
echo "build folder created." echo "build folder created."
fi fi
cmake -G "Unix Makefiles" -D CMAKE_CXX_COMPILER=/usr/bin/g++ -S . -B ./build/ cmake -G "Unix Makefiles" -D CMAKE_CXX_COMPILER=/usr/bin/g++ -S . -B ./build/
if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
cmake --build ./build/ --config DEBUG cmake --build ./build/ --config DEBUG
if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
content=$(<"./CMakeLists.txt") content=$(<"./CMakeLists.txt")
exePath="" exePath=""
while IFS= read -r line; do while IFS= read -r line; do
if [[ $line == "set(MY_EXECUTABLE_NAME"* ]]; then if [[ $line == "set(MY_EXECUTABLE_NAME"* ]]; then
pattern="\"([^\"]+)\"" pattern="\"([^\"]+)\""
if [[ $line =~ $pattern ]]; then if [[ $line =~ $pattern ]]; then
contentInParentheses="${BASH_REMATCH[1]}" contentInParentheses="${BASH_REMATCH[1]}"
result=($contentInParentheses) result=($contentInParentheses)
exePath="./build/bin/${result[0]}" exePath="./build/bin/${result[0]}"
echo "start running as follows..." echo "start running as follows..."
echo "==================================================" echo "=================================================="
fi fi
fi fi
done <<<"$content" done <<<"$content"
# execute the binary file # execute the binary file
if [ -n "$exePath" ]; then if [ -n "$exePath" ]; then
$exePath $exePath
else else
echo "cannot find executable file path" echo "cannot find executable file path"
fi fi
fi fi
fi fi

36
lrun.sh → command/scripts/lrun.sh Executable file → Normal file
View File

@ -1,18 +1,18 @@
content=$(<"./CMakeLists.txt") content=$(<"./CMakeLists.txt")
exePath="" exePath=""
while IFS= read -r line; do while IFS= read -r line; do
if [[ $line == "set(MY_EXECUTABLE_NAME"* ]]; then if [[ $line == "set(MY_EXECUTABLE_NAME"* ]]; then
pattern="\"([^\"]+)\"" pattern="\"([^\"]+)\""
if [[ $line =~ $pattern ]]; then if [[ $line =~ $pattern ]]; then
contentInParentheses="${BASH_REMATCH[1]}" contentInParentheses="${BASH_REMATCH[1]}"
result=($contentInParentheses) result=($contentInParentheses)
exePath="./build/bin/${result[0]}" exePath="./build/bin/${result[0]}"
fi fi
fi fi
done <<<"$content" done <<<"$content"
if [ -n "$exePath" ]; then if [ -n "$exePath" ]; then
$exePath $exePath
else else
echo "cannot find executable file path" echo "cannot find executable file path"
fi fi

26
scripts/lcompile.ps1 Normal file
View File

@ -0,0 +1,26 @@
# generate compile to exe files
$currentDirectory = Get-Location
$cmakeListsPath = Join-Path -Path $currentDirectory -ChildPath "CMakeLists.txt"
if (-not (Test-Path $cmakeListsPath))
{
Write-Host("No CMakeLists.txt in current directory, please check.")
return
}
Write-Host "Start generating and compiling..."
$buildFolderPath = ".\build"
if (-not (Test-Path $buildFolderPath))
{
New-Item -ItemType Directory -Path $buildFolderPath | Out-Null
Write-Host "build folder created."
}
cmake -G "Visual Studio 17 2022" -A x64 -S . -B ./build/
if ($LASTEXITCODE -eq 0)
{
cmake --build ./build/ --config DEBUG
}

46
lcompile.sh → scripts/lcompile.sh Executable file → Normal file
View File

@ -1,23 +1,23 @@
#!/bin/bash #!/bin/bash
currentDirectory=$(pwd) currentDirectory=$(pwd)
cmakeListsPath="${currentDirectory}/CMakeLists.txt" cmakeListsPath="${currentDirectory}/CMakeLists.txt"
if [ ! -f "$cmakeListsPath" ]; then if [ ! -f "$cmakeListsPath" ]; then
echo "No CMakeLists.txt in current directory, please check." echo "No CMakeLists.txt in current directory, please check."
exit 1 exit 1
fi fi
echo "Start generating and compiling..." echo "Start generating and compiling..."
buildFolderPath="./build" buildFolderPath="./build"
if [ ! -d "$buildFolderPath" ]; then if [ ! -d "$buildFolderPath" ]; then
mkdir -p "$buildFolderPath" mkdir -p "$buildFolderPath"
echo "build folder created." echo "build folder created."
fi fi
cmake -G "Unix Makefiles" -D CMAKE_CXX_COMPILER=/usr/bin/g++ -S . -B ./build/ cmake -G "Unix Makefiles" -D CMAKE_CXX_COMPILER=/usr/bin/g++ -S . -B ./build/
if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
cmake --build ./build/ --config DEBUG cmake --build ./build/ --config DEBUG
fi fi

45
scripts/llaunch.ps1 Normal file
View File

@ -0,0 +1,45 @@
#
# generate, compile and run exe files
#
function getExePathFromCMakeLists() {
$content = Get-Content -Raw -Path "./CMakeLists.txt"
$exePath = ""
foreach ($line in $content -split "`n") {
if ($line -match 'set\(MY_EXECUTABLE_NAME[^\"]*\"([^\"]+)\"') {
$exeName = $matches[1]
$exePath = "./build/bin/Debug/$exeName" + ".exe"
break
}
}
return $exePath
}
$currentDirectory = Get-Location
$cmakeListsPath = Join-Path -Path $currentDirectory -ChildPath "CMakeLists.txt"
if (-not (Test-Path $cmakeListsPath)) {
Write-Host("No CMakeLists.txt in current directory, please check.")
return
}
Write-Host "Start generating and compiling..."
$buildFolderPath = ".\build"
if (-not (Test-Path $buildFolderPath)) {
New-Item -ItemType Directory -Path $buildFolderPath | Out-Null
Write-Host "build folder created."
}
cmake -G "Visual Studio 17 2022" -A x64 -S . -B ./build/
if ($LASTEXITCODE -eq 0) {
cmake --build ./build/ --config DEBUG
if ($LASTEXITCODE -eq 0) {
$exePath = getExePathFromCMakeLists
Write-Host "start running as follows..."
Write-Host "=================================================="
Invoke-Expression $exePath
}
}

90
command/llaunch.sh → scripts/llaunch.sh Executable file → Normal file
View File

@ -1,45 +1,45 @@
#!/bin/bash #!/bin/bash
currentDirectory=$(pwd) currentDirectory=$(pwd)
cmakeListsPath="${currentDirectory}/CMakeLists.txt" cmakeListsPath="${currentDirectory}/CMakeLists.txt"
if [ ! -f "$cmakeListsPath" ]; then if [ ! -f "$cmakeListsPath" ]; then
echo "No CMakeLists.txt in current directory, please check." echo "No CMakeLists.txt in current directory, please check."
exit 1 exit 1
fi fi
echo "Start generating and compiling..." echo "Start generating and compiling..."
buildFolderPath="./build" buildFolderPath="./build"
if [ ! -d "$buildFolderPath" ]; then if [ ! -d "$buildFolderPath" ]; then
mkdir -p "$buildFolderPath" mkdir -p "$buildFolderPath"
echo "build folder created." echo "build folder created."
fi fi
cmake -G "Unix Makefiles" -D CMAKE_CXX_COMPILER=/usr/bin/g++ -S . -B ./build/ cmake -G "Unix Makefiles" -D CMAKE_CXX_COMPILER=/usr/bin/g++ -S . -B ./build/
if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
cmake --build ./build/ --config DEBUG cmake --build ./build/ --config DEBUG
if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
content=$(<"./CMakeLists.txt") content=$(<"./CMakeLists.txt")
exePath="" exePath=""
while IFS= read -r line; do while IFS= read -r line; do
if [[ $line == "set(MY_EXECUTABLE_NAME"* ]]; then if [[ $line == "set(MY_EXECUTABLE_NAME"* ]]; then
pattern="\"([^\"]+)\"" pattern="\"([^\"]+)\""
if [[ $line =~ $pattern ]]; then if [[ $line =~ $pattern ]]; then
contentInParentheses="${BASH_REMATCH[1]}" contentInParentheses="${BASH_REMATCH[1]}"
result=($contentInParentheses) result=($contentInParentheses)
exePath="./build/bin/${result[0]}" exePath="./build/bin/${result[0]}"
echo "start running as follows..." echo "start running as follows..."
echo "==================================================" echo "=================================================="
fi fi
fi fi
done <<<"$content" done <<<"$content"
# execute the binary file # execute the binary file
if [ -n "$exePath" ]; then if [ -n "$exePath" ]; then
$exePath $exePath
else else
echo "cannot find executable file path" echo "cannot find executable file path"
fi fi
fi fi
fi fi

20
scripts/lrun.ps1 Normal file
View File

@ -0,0 +1,20 @@
#
# run exe file that has already been compiled before
#
function getExePathFromCMakeLists() {
$content = Get-Content -Raw -Path "./CMakeLists.txt"
$exePath = ""
foreach ($line in $content -split "`n") {
if ($line -match 'set\(MY_EXECUTABLE_NAME[^\"]*\"([^\"]+)\"') {
$exeName = $matches[1]
$exePath = "./build/bin/Debug/$exeName" + ".exe"
break
}
}
return $exePath
}
$exePath = getExePathFromCMakeLists
#Write-Host "start running as follows..."
#Write-Host "=================================================="
Invoke-Expression $exePath

36
command/lrun.sh → scripts/lrun.sh Executable file → Normal file
View File

@ -1,18 +1,18 @@
content=$(<"./CMakeLists.txt") content=$(<"./CMakeLists.txt")
exePath="" exePath=""
while IFS= read -r line; do while IFS= read -r line; do
if [[ $line == "set(MY_EXECUTABLE_NAME"* ]]; then if [[ $line == "set(MY_EXECUTABLE_NAME"* ]]; then
pattern="\"([^\"]+)\"" pattern="\"([^\"]+)\""
if [[ $line =~ $pattern ]]; then if [[ $line =~ $pattern ]]; then
contentInParentheses="${BASH_REMATCH[1]}" contentInParentheses="${BASH_REMATCH[1]}"
result=($contentInParentheses) result=($contentInParentheses)
exePath="./build/bin/${result[0]}" exePath="./build/bin/${result[0]}"
fi fi
fi fi
done <<<"$content" done <<<"$content"
if [ -n "$exePath" ]; then if [ -n "$exePath" ]; then
$exePath $exePath
else else
echo "cannot find executable file path" echo "cannot find executable file path"
fi fi

786
src/include/userdict.h
View File

@ -1,390 +1,396 @@
/* /*
* Copyright (C) 2009 The Android Open Source Project * Copyright (C) 2009 The Android Open Source Project
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
* *
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* *
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
#ifndef PINYINIME_INCLUDE_USERDICT_H__ #ifndef PINYINIME_INCLUDE_USERDICT_H__
#define PINYINIME_INCLUDE_USERDICT_H__ #define PINYINIME_INCLUDE_USERDICT_H__
#define ___CACHE_ENABLED___ #define ___CACHE_ENABLED___
#define ___SYNC_ENABLED___ #define ___SYNC_ENABLED___
#define ___PREDICT_ENABLED___ #define ___PREDICT_ENABLED___
// Debug performance for operations // Debug performance for operations
// #define ___DEBUG_PERF___ // #define ___DEBUG_PERF___
#include <pthread.h> #ifdef _WIN32
#include "atomdictbase.h" #include <time.h>
#include <winsock.h> // timeval
namespace ime_pinyin { #else
#include <pthread.h>
class UserDict : public AtomDictBase { #include <sys/time.h>
public: #endif
UserDict(); #include "atomdictbase.h"
~UserDict();
namespace ime_pinyin {
bool load_dict(const char *file_name, LemmaIdType start_id, LemmaIdType end_id);
class UserDict : public AtomDictBase {
bool close_dict(); public:
UserDict();
size_t number_of_lemmas(); ~UserDict();
void reset_milestones(uint16 from_step, MileStoneHandle from_handle); bool load_dict(const char *file_name, LemmaIdType start_id, LemmaIdType end_id);
MileStoneHandle extend_dict(MileStoneHandle from_handle, const DictExtPara *dep, LmaPsbItem *lpi_items, size_t lpi_max, size_t *lpi_num); bool close_dict();
size_t get_lpis(const uint16 *splid_str, uint16 splid_str_len, LmaPsbItem *lpi_items, size_t lpi_max); size_t number_of_lemmas();
uint16 get_lemma_str(LemmaIdType id_lemma, char16 *str_buf, uint16 str_max); void reset_milestones(uint16 from_step, MileStoneHandle from_handle);
uint16 get_lemma_splids(LemmaIdType id_lemma, uint16 *splids, uint16 splids_max, bool arg_valid); MileStoneHandle extend_dict(MileStoneHandle from_handle, const DictExtPara *dep, LmaPsbItem *lpi_items, size_t lpi_max, size_t *lpi_num);
size_t predict(const char16 last_hzs[], uint16 hzs_len, NPredictItem *npre_items, size_t npre_max, size_t b4_used); size_t get_lpis(const uint16 *splid_str, uint16 splid_str_len, LmaPsbItem *lpi_items, size_t lpi_max);
// Full spelling ids are required uint16 get_lemma_str(LemmaIdType id_lemma, char16 *str_buf, uint16 str_max);
LemmaIdType put_lemma(char16 lemma_str[], uint16 splids[], uint16 lemma_len, uint16 count);
uint16 get_lemma_splids(LemmaIdType id_lemma, uint16 *splids, uint16 splids_max, bool arg_valid);
LemmaIdType update_lemma(LemmaIdType lemma_id, int16 delta_count, bool selected);
size_t predict(const char16 last_hzs[], uint16 hzs_len, NPredictItem *npre_items, size_t npre_max, size_t b4_used);
LemmaIdType get_lemma_id(char16 lemma_str[], uint16 splids[], uint16 lemma_len);
// Full spelling ids are required
LmaScoreType get_lemma_score(LemmaIdType lemma_id); LemmaIdType put_lemma(char16 lemma_str[], uint16 splids[], uint16 lemma_len, uint16 count);
LmaScoreType get_lemma_score(char16 lemma_str[], uint16 splids[], uint16 lemma_len); LemmaIdType update_lemma(LemmaIdType lemma_id, int16 delta_count, bool selected);
bool remove_lemma(LemmaIdType lemma_id); LemmaIdType get_lemma_id(char16 lemma_str[], uint16 splids[], uint16 lemma_len);
size_t get_total_lemma_count(); LmaScoreType get_lemma_score(LemmaIdType lemma_id);
void set_total_lemma_count_of_others(size_t count);
LmaScoreType get_lemma_score(char16 lemma_str[], uint16 splids[], uint16 lemma_len);
void flush_cache();
bool remove_lemma(LemmaIdType lemma_id);
void set_limit(uint32 max_lemma_count, uint32 max_lemma_size, uint32 reclaim_ratio);
size_t get_total_lemma_count();
void reclaim(); void set_total_lemma_count_of_others(size_t count);
void defragment(); void flush_cache();
#ifdef ___SYNC_ENABLED___ void set_limit(uint32 max_lemma_count, uint32 max_lemma_size, uint32 reclaim_ratio);
void clear_sync_lemmas(unsigned int start, unsigned int end);
void reclaim();
int get_sync_count();
void defragment();
LemmaIdType put_lemma_no_sync(char16 lemma_str[], uint16 splids[], uint16 lemma_len, uint16 count, uint64 lmt);
/** #ifdef ___SYNC_ENABLED___
* Add lemmas encoded in UTF-16LE into dictionary without adding sync flag. void clear_sync_lemmas(unsigned int start, unsigned int end);
*
* @param lemmas in format of 'wo men,WM,0.32;da jia,DJ,0.12' int get_sync_count();
* @param len length of lemmas string in UTF-16LE
* @return newly added lemma count LemmaIdType put_lemma_no_sync(char16 lemma_str[], uint16 splids[], uint16 lemma_len, uint16 count, uint64 lmt);
*/ /**
int put_lemmas_no_sync_from_utf16le_string(char16 *lemmas, int len); * Add lemmas encoded in UTF-16LE into dictionary without adding sync flag.
*
/** * @param lemmas in format of 'wo men,WM,0.32;da jia,DJ,0.12'
* Get lemmas need sync to a UTF-16LE string of above format. * @param len length of lemmas string in UTF-16LE
* Note: input buffer (str) must not be too small. If str is too small to * @return newly added lemma count
* contain single one lemma, there might be a dead loop. */
* int put_lemmas_no_sync_from_utf16le_string(char16 *lemmas, int len);
* @param str buffer to write lemmas
* @param size buffer size in UTF-16LE /**
* @param count output value of lemma returned * Get lemmas need sync to a UTF-16LE string of above format.
* @return UTF-16LE string length * Note: input buffer (str) must not be too small. If str is too small to
*/ * contain single one lemma, there might be a dead loop.
int get_sync_lemmas_in_utf16le_string_from_beginning(char16 *str, int size, int *count); *
* @param str buffer to write lemmas
#endif * @param size buffer size in UTF-16LE
* @param count output value of lemma returned
struct UserDictStat { * @return UTF-16LE string length
uint32 version; */
const char *file_name; int get_sync_lemmas_in_utf16le_string_from_beginning(char16 *str, int size, int *count);
struct timeval load_time;
struct timeval last_update; #endif
uint32 disk_size;
uint32 lemma_count; struct UserDictStat {
uint32 lemma_size; uint32 version;
uint32 delete_count; const char *file_name;
uint32 delete_size; struct timeval load_time;
#ifdef ___SYNC_ENABLED___ struct timeval last_update;
uint32 sync_count; uint32 disk_size;
#endif uint32 lemma_count;
uint32 reclaim_ratio; uint32 lemma_size;
uint32 limit_lemma_count; uint32 delete_count;
uint32 limit_lemma_size; uint32 delete_size;
}; #ifdef ___SYNC_ENABLED___
uint32 sync_count;
bool state(UserDictStat *stat); #endif
uint32 reclaim_ratio;
private: uint32 limit_lemma_count;
uint32 total_other_nfreq_; uint32 limit_lemma_size;
struct timeval load_time_; };
LemmaIdType start_id_;
uint32 version_; bool state(UserDictStat *stat);
uint8 *lemmas_;
private:
// In-Memory-Only flag for each lemma uint32 total_other_nfreq_;
static const uint8 kUserDictLemmaFlagRemove = 1; struct timeval load_time_;
// Inuse lemmas' offset LemmaIdType start_id_;
uint32 *offsets_; uint32 version_;
// Highest bit in offset tells whether corresponding lemma is removed uint8 *lemmas_;
static const uint32 kUserDictOffsetFlagRemove = (1 << 31);
// Maximum possible for the offset // In-Memory-Only flag for each lemma
static const uint32 kUserDictOffsetMask = ~(kUserDictOffsetFlagRemove); static const uint8 kUserDictLemmaFlagRemove = 1;
// Bit width for last modified time, from 1 to 16 // Inuse lemmas' offset
static const uint32 kUserDictLMTBitWidth = 16; uint32 *offsets_;
// Granularity for last modified time in second // Highest bit in offset tells whether corresponding lemma is removed
static const uint32 kUserDictLMTGranularity = 60 * 60 * 24 * 7; static const uint32 kUserDictOffsetFlagRemove = (1 << 31);
// Maximum frequency count // Maximum possible for the offset
static const uint16 kUserDictMaxFrequency = 0xFFFF; static const uint32 kUserDictOffsetMask = ~(kUserDictOffsetFlagRemove);
// Bit width for last modified time, from 1 to 16
#define COARSE_UTC(year, month, day, hour, minute, second) ((year - 1970) * 365 * 24 * 60 * 60 + (month - 1) * 30 * 24 * 60 * 60 + (day - 1) * 24 * 60 * 60 + (hour - 0) * 60 * 60 + (minute - 0) * 60 + (second - 0)) static const uint32 kUserDictLMTBitWidth = 16;
static const uint64 kUserDictLMTSince = COARSE_UTC(2009, 1, 1, 0, 0, 0); // Granularity for last modified time in second
static const uint32 kUserDictLMTGranularity = 60 * 60 * 24 * 7;
// Correspond to offsets_ // Maximum frequency count
uint32 *scores_; static const uint16 kUserDictMaxFrequency = 0xFFFF;
// Following two fields are only valid in memory
uint32 *ids_; #define COARSE_UTC(year, month, day, hour, minute, second) ((year - 1970) * 365 * 24 * 60 * 60 + (month - 1) * 30 * 24 * 60 * 60 + (day - 1) * 24 * 60 * 60 + (hour - 0) * 60 * 60 + (minute - 0) * 60 + (second - 0))
#ifdef ___PREDICT_ENABLED___ static const uint64 kUserDictLMTSince = COARSE_UTC(2009, 1, 1, 0, 0, 0);
uint32 *predicts_;
#endif // Correspond to offsets_
#ifdef ___SYNC_ENABLED___ uint32 *scores_;
uint32 *syncs_; // Following two fields are only valid in memory
size_t sync_count_size_; uint32 *ids_;
#endif #ifdef ___PREDICT_ENABLED___
uint32 *offsets_by_id_; uint32 *predicts_;
#endif
size_t lemma_count_left_; #ifdef ___SYNC_ENABLED___
size_t lemma_size_left_; uint32 *syncs_;
size_t sync_count_size_;
const char *dict_file_; #endif
uint32 *offsets_by_id_;
// Be sure size is 4xN
struct UserDictInfo { size_t lemma_count_left_;
// When limitation reached, how much percentage will be reclaimed (1 ~ 100) size_t lemma_size_left_;
uint32 reclaim_ratio;
// maximum lemma count, 0 means no limitation const char *dict_file_;
uint32 limit_lemma_count;
// Maximum lemma size, it's different from // Be sure size is 4xN
// whole disk file size or in-mem dict size struct UserDictInfo {
// 0 means no limitation // When limitation reached, how much percentage will be reclaimed (1 ~ 100)
uint32 limit_lemma_size; uint32 reclaim_ratio;
// Total lemma count including deleted and inuse // maximum lemma count, 0 means no limitation
// Also indicate offsets_ size uint32 limit_lemma_count;
uint32 lemma_count; // Maximum lemma size, it's different from
// Total size of lemmas including used and freed // whole disk file size or in-mem dict size
uint32 lemma_size; // 0 means no limitation
// Freed lemma count uint32 limit_lemma_size;
uint32 free_count; // Total lemma count including deleted and inuse
// Freed lemma size in byte // Also indicate offsets_ size
uint32 free_size; uint32 lemma_count;
#ifdef ___SYNC_ENABLED___ // Total size of lemmas including used and freed
uint32 sync_count; uint32 lemma_size;
#endif // Freed lemma count
int32 total_nfreq; uint32 free_count;
} dict_info_; // Freed lemma size in byte
uint32 free_size;
static const uint32 kUserDictVersion = 0x0ABCDEF0; #ifdef ___SYNC_ENABLED___
uint32 sync_count;
static const uint32 kUserDictPreAlloc = 32; #endif
static const uint32 kUserDictAverageNchar = 8; int32 total_nfreq;
} dict_info_;
enum UserDictState {
// Keep in order static const uint32 kUserDictVersion = 0x0ABCDEF0;
USER_DICT_NONE = 0,
USER_DICT_SYNC, static const uint32 kUserDictPreAlloc = 32;
#ifdef ___SYNC_ENABLED___ static const uint32 kUserDictAverageNchar = 8;
USER_DICT_SYNC_DIRTY,
#endif enum UserDictState {
USER_DICT_SCORE_DIRTY, // Keep in order
USER_DICT_OFFSET_DIRTY, USER_DICT_NONE = 0,
USER_DICT_LEMMA_DIRTY, USER_DICT_SYNC,
#ifdef ___SYNC_ENABLED___
USER_DICT_DEFRAGMENTED, USER_DICT_SYNC_DIRTY,
} state_; #endif
USER_DICT_SCORE_DIRTY,
struct UserDictSearchable { USER_DICT_OFFSET_DIRTY,
uint16 splids_len; USER_DICT_LEMMA_DIRTY,
uint16 splid_start[kMaxLemmaSize];
uint16 splid_count[kMaxLemmaSize]; USER_DICT_DEFRAGMENTED,
// Compact inital letters for both FuzzyCompareSpellId and cache system } state_;
uint32 signature[kMaxLemmaSize / 4];
}; struct UserDictSearchable {
uint16 splids_len;
#ifdef ___CACHE_ENABLED___ uint16 splid_start[kMaxLemmaSize];
enum UserDictCacheType { uint16 splid_count[kMaxLemmaSize];
USER_DICT_CACHE, // Compact inital letters for both FuzzyCompareSpellId and cache system
USER_DICT_MISS_CACHE, uint32 signature[kMaxLemmaSize / 4];
}; };
static const int kUserDictCacheSize = 4; #ifdef ___CACHE_ENABLED___
static const int kUserDictMissCacheSize = kMaxLemmaSize - 1; enum UserDictCacheType {
USER_DICT_CACHE,
struct UserDictMissCache { USER_DICT_MISS_CACHE,
uint32 signatures[kUserDictMissCacheSize][kMaxLemmaSize / 4]; };
uint16 head, tail;
} miss_caches_[kMaxLemmaSize]; static const int kUserDictCacheSize = 4;
static const int kUserDictMissCacheSize = kMaxLemmaSize - 1;
struct UserDictCache {
uint32 signatures[kUserDictCacheSize][kMaxLemmaSize / 4]; struct UserDictMissCache {
uint32 offsets[kUserDictCacheSize]; uint32 signatures[kUserDictMissCacheSize][kMaxLemmaSize / 4];
uint32 lengths[kUserDictCacheSize]; uint16 head, tail;
// Ring buffer } miss_caches_[kMaxLemmaSize];
uint16 head, tail;
} caches_[kMaxLemmaSize]; struct UserDictCache {
uint32 signatures[kUserDictCacheSize][kMaxLemmaSize / 4];
void cache_init(); uint32 offsets[kUserDictCacheSize];
uint32 lengths[kUserDictCacheSize];
void cache_push(UserDictCacheType type, UserDictSearchable *searchable, uint32 offset, uint32 length); // Ring buffer
uint16 head, tail;
bool cache_hit(UserDictSearchable *searchable, uint32 *offset, uint32 *length); } caches_[kMaxLemmaSize];
bool load_cache(UserDictSearchable *searchable, uint32 *offset, uint32 *length); void cache_init();
void save_cache(UserDictSearchable *searchable, uint32 offset, uint32 length); void cache_push(UserDictCacheType type, UserDictSearchable *searchable, uint32 offset, uint32 length);
void reset_cache(); bool cache_hit(UserDictSearchable *searchable, uint32 *offset, uint32 *length);
bool load_miss_cache(UserDictSearchable *searchable); bool load_cache(UserDictSearchable *searchable, uint32 *offset, uint32 *length);
void save_miss_cache(UserDictSearchable *searchable); void save_cache(UserDictSearchable *searchable, uint32 offset, uint32 length);
void reset_miss_cache(); void reset_cache();
#endif
bool load_miss_cache(UserDictSearchable *searchable);
LmaScoreType translate_score(int f);
void save_miss_cache(UserDictSearchable *searchable);
int extract_score_freq(int raw_score);
void reset_miss_cache();
uint64 extract_score_lmt(int raw_score); #endif
inline int build_score(uint64 lmt, int freq); LmaScoreType translate_score(int f);
inline int64 utf16le_atoll(uint16 *s, int len); int extract_score_freq(int raw_score);
inline int utf16le_lltoa(int64 v, uint16 *s, int size); uint64 extract_score_lmt(int raw_score);
LemmaIdType _put_lemma(char16 lemma_str[], uint16 splids[], uint16 lemma_len, uint16 count, uint64 lmt); inline int build_score(uint64 lmt, int freq);
size_t _get_lpis(const uint16 *splid_str, uint16 splid_str_len, LmaPsbItem *lpi_items, size_t lpi_max, bool *need_extend); inline int64 utf16le_atoll(uint16 *s, int len);
int _get_lemma_score(char16 lemma_str[], uint16 splids[], uint16 lemma_len); inline int utf16le_lltoa(int64 v, uint16 *s, int size);
int _get_lemma_score(LemmaIdType lemma_id); LemmaIdType _put_lemma(char16 lemma_str[], uint16 splids[], uint16 lemma_len, uint16 count, uint64 lmt);
int is_fuzzy_prefix_spell_id(const uint16 *id1, uint16 len1, const UserDictSearchable *searchable); size_t _get_lpis(const uint16 *splid_str, uint16 splid_str_len, LmaPsbItem *lpi_items, size_t lpi_max, bool *need_extend);
bool is_prefix_spell_id(const uint16 *fullids, uint16 fulllen, const UserDictSearchable *searchable); int _get_lemma_score(char16 lemma_str[], uint16 splids[], uint16 lemma_len);
uint32 get_dict_file_size(UserDictInfo *info); int _get_lemma_score(LemmaIdType lemma_id);
bool reset(const char *file); int is_fuzzy_prefix_spell_id(const uint16 *id1, uint16 len1, const UserDictSearchable *searchable);
bool validate(const char *file); bool is_prefix_spell_id(const uint16 *fullids, uint16 fulllen, const UserDictSearchable *searchable);
bool load(const char *file, LemmaIdType start_id); uint32 get_dict_file_size(UserDictInfo *info);
bool is_valid_state(); bool reset(const char *file);
bool is_valid_lemma_id(LemmaIdType id); bool validate(const char *file);
LemmaIdType get_max_lemma_id(); bool load(const char *file, LemmaIdType start_id);
void set_lemma_flag(uint32 offset, uint8 flag); bool is_valid_state();
char get_lemma_flag(uint32 offset); bool is_valid_lemma_id(LemmaIdType id);
char get_lemma_nchar(uint32 offset); LemmaIdType get_max_lemma_id();
uint16 *get_lemma_spell_ids(uint32 offset); void set_lemma_flag(uint32 offset, uint8 flag);
uint16 *get_lemma_word(uint32 offset); char get_lemma_flag(uint32 offset);
// Prepare searchable to fasten locate process char get_lemma_nchar(uint32 offset);
void prepare_locate(UserDictSearchable *searchable, const uint16 *splids, uint16 len);
uint16 *get_lemma_spell_ids(uint32 offset);
// Compare initial letters only
int32 fuzzy_compare_spell_id(const uint16 *id1, uint16 len1, const UserDictSearchable *searchable); uint16 *get_lemma_word(uint32 offset);
// Compare exactly two spell ids // Prepare searchable to fasten locate process
// First argument must be a full id spell id void prepare_locate(UserDictSearchable *searchable, const uint16 *splids, uint16 len);
bool equal_spell_id(const uint16 *fullids, uint16 fulllen, const UserDictSearchable *searchable);
// Compare initial letters only
// Find first item by initial letters int32 fuzzy_compare_spell_id(const uint16 *id1, uint16 len1, const UserDictSearchable *searchable);
int32 locate_first_in_offsets(const UserDictSearchable *searchable);
// Compare exactly two spell ids
LemmaIdType append_a_lemma(char16 lemma_str[], uint16 splids[], uint16 lemma_len, uint16 count, uint64 lmt); // First argument must be a full id spell id
bool equal_spell_id(const uint16 *fullids, uint16 fulllen, const UserDictSearchable *searchable);
// Check if a lemma is in dictionary
int32 locate_in_offsets(char16 lemma_str[], uint16 splid_str[], uint16 lemma_len); // Find first item by initial letters
int32 locate_first_in_offsets(const UserDictSearchable *searchable);
bool remove_lemma_by_offset_index(int offset_index);
#ifdef ___PREDICT_ENABLED___ LemmaIdType append_a_lemma(char16 lemma_str[], uint16 splids[], uint16 lemma_len, uint16 count, uint64 lmt);
uint32 locate_where_to_insert_in_predicts(const uint16 *words, int lemma_len);
// Check if a lemma is in dictionary
int32 locate_first_in_predicts(const uint16 *words, int lemma_len); int32 locate_in_offsets(char16 lemma_str[], uint16 splid_str[], uint16 lemma_len);
void remove_lemma_from_predict_list(uint32 offset); bool remove_lemma_by_offset_index(int offset_index);
#endif #ifdef ___PREDICT_ENABLED___
#ifdef ___SYNC_ENABLED___ uint32 locate_where_to_insert_in_predicts(const uint16 *words, int lemma_len);
void queue_lemma_for_sync(LemmaIdType id);
int32 locate_first_in_predicts(const uint16 *words, int lemma_len);
void remove_lemma_from_sync_list(uint32 offset);
void remove_lemma_from_predict_list(uint32 offset);
void write_back_sync(int fd); #endif
#endif #ifdef ___SYNC_ENABLED___
void write_back_score(int fd); void queue_lemma_for_sync(LemmaIdType id);
void write_back_offset(int fd);
void write_back_lemma(int fd); void remove_lemma_from_sync_list(uint32 offset);
void write_back_all(int fd);
void write_back(); void write_back_sync(int fd);
#endif
struct UserDictScoreOffsetPair { void write_back_score(int fd);
int score; void write_back_offset(int fd);
uint32 offset_index; void write_back_lemma(int fd);
}; void write_back_all(int fd);
void write_back();
inline void swap(UserDictScoreOffsetPair *sop, int i, int j);
struct UserDictScoreOffsetPair {
void shift_down(UserDictScoreOffsetPair *sop, int i, int n); int score;
uint32 offset_index;
// On-disk format for each lemma };
// +-------------+
// | Version (4) | inline void swap(UserDictScoreOffsetPair *sop, int i, int j);
// +-------------+
// +-----------+-----------+--------------------+-------------------+ void shift_down(UserDictScoreOffsetPair *sop, int i, int n);
// | Spare (1) | Nchar (1) | Splids (2 x Nchar) | Lemma (2 x Nchar) |
// +-----------+-----------+--------------------+-------------------+ // On-disk format for each lemma
// ... // +-------------+
// +-----------------------+ +-------------+ <---Offset of offset // | Version (4) |
// | Offset1 by_splids (4) | ... | OffsetN (4) | // +-------------+
// +-----------------------+ +-------------+ // +-----------+-----------+--------------------+-------------------+
#ifdef ___PREDICT_ENABLED___ // | Spare (1) | Nchar (1) | Splids (2 x Nchar) | Lemma (2 x Nchar) |
// +----------------------+ +-------------+ // +-----------+-----------+--------------------+-------------------+
// | Offset1 by_lemma (4) | ... | OffsetN (4) | // ...
// +----------------------+ +-------------+ // +-----------------------+ +-------------+ <---Offset of offset
#endif // | Offset1 by_splids (4) | ... | OffsetN (4) |
// +------------+ +------------+ // +-----------------------+ +-------------+
// | Score1 (4) | ... | ScoreN (4) | #ifdef ___PREDICT_ENABLED___
// +------------+ +------------+ // +----------------------+ +-------------+
#ifdef ___SYNC_ENABLED___ // | Offset1 by_lemma (4) | ... | OffsetN (4) |
// +-------------+ +-------------+ // +----------------------+ +-------------+
// | NewAdd1 (4) | ... | NewAddN (4) | #endif
// +-------------+ +-------------+ // +------------+ +------------+
#endif // | Score1 (4) | ... | ScoreN (4) |
// +----------------+ // +------------+ +------------+
// | Dict Info (4x) | #ifdef ___SYNC_ENABLED___
// +----------------+ // +-------------+ +-------------+
}; // | NewAdd1 (4) | ... | NewAddN (4) |
} // namespace ime_pinyin // +-------------+ +-------------+
#endif
#endif // +----------------+
// | Dict Info (4x) |
// +----------------+
};
} // namespace ime_pinyin
#endif

4175
src/share/userdict.cpp
View File

File diff suppressed because it is too large Load Diff

70
tests/main.cpp
View File

@ -1,35 +1,35 @@
#include "../src/include/pinyinime.h" #include "../src/include/pinyinime.h"
#include <codecvt> #include <codecvt>
#include <iostream> #include <iostream>
#include <locale> #include <locale>
#include <string> #include <string>
std::string fromUtf16(const ime_pinyin::char16 *buf, size_t len) { std::string fromUtf16(const ime_pinyin::char16 *buf, size_t len) {
// 转换为标准 char16_t // 转换为标准 char16_t
std::u16string utf16Str(reinterpret_cast<const char16_t *>(buf), len); std::u16string utf16Str(reinterpret_cast<const char16_t *>(buf), len);
std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> convert; std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> convert;
return convert.to_bytes(utf16Str); return convert.to_bytes(utf16Str);
} }
int main() { int main() {
if (!ime_pinyin::im_open_decoder("./data/dict_pinyin.dat", "./data/user_dict.dat")) { if (!ime_pinyin::im_open_decoder("./data/dict_pinyin.dat", "./data/user_dict.dat")) {
std::cout << "fany bug.\n"; std::cout << "fany bug.\n";
return 0; return 0;
} }
std::string pinyin = "ni'ma'si'le"; std::string pinyin = "ni'ma'si'le";
pinyin = "ni'ma'mei'si"; pinyin = "ni'ma'mei'si";
pinyin = "ni'shuo'ni'ma'ne"; pinyin = "ni'shuo'ni'ma'ne";
size_t cand_cnt = ime_pinyin::im_search(pinyin.c_str(), pinyin.size()); size_t cand_cnt = ime_pinyin::im_search(pinyin.c_str(), pinyin.size());
ime_pinyin::char16 buf[256] = {0}; ime_pinyin::char16 buf[256] = {0};
std::string msg; std::string msg;
for (size_t i = 0; i < 100; ++i) { for (size_t i = 0; i < 100; ++i) {
ime_pinyin::im_get_candidate(i, buf, 255); ime_pinyin::im_get_candidate(i, buf, 255);
size_t len = 0; size_t len = 0;
while (buf[len] != 0 && len < 255) ++len; while (buf[len] != 0 && len < 255) ++len;
msg.append(fromUtf16(buf, len) + " "); msg.append(fromUtf16(buf, len) + " ");
} }
std::cout << "候选项数量: " << cand_cnt << std::endl; std::cout << "候选项数量: " << cand_cnt << std::endl;
std::cout << "候选项本体: " << msg << std::endl; std::cout << "候选项本体: " << msg << std::endl;
return 0; return 0;
} }