diff --git a/.clangd b/.clangd index 7f04681..58dc6fa 100644 --- a/.clangd +++ b/.clangd @@ -1,8 +1,11 @@ CompileFlags: Add: [ - "-std=c++17", - # -Wno-unused-value, - ] # ignore some specific warnings - CompilationDatabase: build/ # Search build/ directory for compile_commands.json + -DUNICODE, + -D_UNICODE, + "-std=c++17", + "-IC:/Users/19833/EDisk/CppCodes/ImeCodes/googlepinyinime-rev/build/vcpkg_installed/x64-windows/include", + # -Wno-unused-value, + ] # ignore some specific warnings + CompilationDatabase: build/ # Search build/ directory for compile_commands.json # Diagnostics: # UnusedIncludes: None # ignore unused includes diff --git a/CMakeLists.txt b/CMakeLists.txt index 920865b..3d63475 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,13 +4,24 @@ project(pinyinime VERSION 1.0 LANGUAGES CXX) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src/include) -set(CMAKE_CXX_STANDARD 11) -set(CMAKE_CXX_STANDARD_REQUIRED True) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_C_STANDARD 99) -set(CMAKE_C_STANDARD_REQUIRED True) +set(CMAKE_C_STANDARD_REQUIRED ON) + +add_definitions( + /D_UNICODE=1 + /DUNICODE=1 +) + +# For utf8cpp, make it use C++11 and later +add_compile_options(/Zc:__cplusplus) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +# For pure-header libs installed by vcpkg like utf8cpp +include_directories("./build/vcpkg_installed/x64-windows/include") + set(HEADERS ./src/include/atomdictbase.h ./src/include/dictbuilder.h diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 0000000..277515d --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,17 @@ +{ + "version": 2, + "configurePresets": [ + { + "name": "vcpkg", + "generator": "Visual Studio 17 2022", + "binaryDir": "${sourceDir}/build", + "environment": { + "VCPKG_ROOT": "C:/Users/19833/scoop/apps/vcpkg/current/" + }, + "cacheVariables": { + "CMAKE_TOOLCHAIN_FILE": "C:/Users/19833/scoop/apps/vcpkg/current/scripts/buildsystems/vcpkg.cmake", + "CMAKE_BUILD_TYPE": "Debug" + } + } + ] +} \ No newline at end of file diff --git a/CMakeUserPresets.json b/CMakeUserPresets.json new file mode 100644 index 0000000..c8b022d --- /dev/null +++ b/CMakeUserPresets.json @@ -0,0 +1,11 @@ +{ + "version": 2, + "configurePresets": [ + { + "name": "default", + "inherits": "vcpkg", + "generator": "Visual Studio 17 2022", + "binaryDir": "${sourceDir}/build" + } + ] +} \ No newline at end of file diff --git a/scripts/config_files/.clangd b/scripts/config_files/.clangd new file mode 100644 index 0000000..58dc6fa --- /dev/null +++ b/scripts/config_files/.clangd @@ -0,0 +1,11 @@ +CompileFlags: + Add: [ + -DUNICODE, + -D_UNICODE, + "-std=c++17", + "-IC:/Users/19833/EDisk/CppCodes/ImeCodes/googlepinyinime-rev/build/vcpkg_installed/x64-windows/include", + # -Wno-unused-value, + ] # ignore some specific warnings + CompilationDatabase: build/ # Search build/ directory for compile_commands.json +# Diagnostics: +# UnusedIncludes: None # ignore unused includes diff --git a/scripts/config_files/CMakeLists.txt b/scripts/config_files/CMakeLists.txt new file mode 100644 index 0000000..99db411 --- /dev/null +++ b/scripts/config_files/CMakeLists.txt @@ -0,0 +1,68 @@ +cmake_minimum_required(VERSION 3.15) + +project(pinyinime VERSION 1.0 LANGUAGES CXX) + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src/include) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_C_STANDARD 99) +set(CMAKE_C_STANDARD_REQUIRED ON) + +add_definitions( + /D_UNICODE=1 + /DUNICODE=1 +) + +# For utf8cpp, make it use C++11 and later +add_compile_options(/Zc:__cplusplus) + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + +# For pure-header libs installed by vcpkg like utf8cpp +include_directories("./build/vcpkg_installed/x64-windows/include") + +set(HEADERS + ./src/include/atomdictbase.h + ./src/include/dictbuilder.h + ./src/include/dictdef.h + ./src/include/dictlist.h + ./src/include/dicttrie.h + ./src/include/lpicache.h + ./src/include/matrixsearch.h + ./src/include/mystdlib.h + ./src/include/ngram.h + ./src/include/pinyinime.h + ./src/include/searchutility.h + ./src/include/spellingtable.h + ./src/include/spellingtrie.h + ./src/include/splparser.h + ./src/include/sync.h + ./src/include/userdict.h + ./src/include/utf16char.h + ./src/include/utf16reader.h +) + +set(SOURCES + ./src/share/dictbuilder.cpp + ./src/share/dictlist.cpp + ./src/share/dicttrie.cpp + ./src/share/lpicache.cpp + ./src/share/matrixsearch.cpp + ./src/share/mystdlib.cpp + ./src/share/ngram.cpp + ./src/share/pinyinime.cpp + ./src/share/searchutility.cpp + ./src/share/spellingtable.cpp + ./src/share/spellingtrie.cpp + ./src/share/splparser.cpp + ./src/share/sync.cpp + ./src/share/userdict.cpp + ./src/share/utf16char.cpp + ./src/share/utf16reader.cpp + ./tests/main.cpp +) + +set(MY_EXECUTABLE_NAME "pinyinime") + +add_executable(pinyinime ${SOURCES} ${HEADERS}) \ No newline at end of file diff --git a/scripts/config_files/CMakePresets.json b/scripts/config_files/CMakePresets.json new file mode 100644 index 0000000..277515d --- /dev/null +++ b/scripts/config_files/CMakePresets.json @@ -0,0 +1,17 @@ +{ + "version": 2, + "configurePresets": [ + { + "name": "vcpkg", + "generator": "Visual Studio 17 2022", + "binaryDir": "${sourceDir}/build", + "environment": { + "VCPKG_ROOT": "C:/Users/19833/scoop/apps/vcpkg/current/" + }, + "cacheVariables": { + "CMAKE_TOOLCHAIN_FILE": "C:/Users/19833/scoop/apps/vcpkg/current/scripts/buildsystems/vcpkg.cmake", + "CMAKE_BUILD_TYPE": "Debug" + } + } + ] +} \ No newline at end of file diff --git a/scripts/lcompile.ps1 b/scripts/lcompile.ps1 index 81d154f..9a59f8b 100644 --- a/scripts/lcompile.ps1 +++ b/scripts/lcompile.ps1 @@ -2,25 +2,23 @@ $currentDirectory = Get-Location $cmakeListsPath = Join-Path -Path $currentDirectory -ChildPath "CMakeLists.txt" -if (-not (Test-Path $cmakeListsPath)) -{ - Write-Host("No CMakeLists.txt in current directory, please check.") - return +if (-not (Test-Path $cmakeListsPath)) { + Write-Host("No CMakeLists.txt in current directory, please check.") + return } Write-Host "Start generating and compiling..." $buildFolderPath = ".\build" -if (-not (Test-Path $buildFolderPath)) -{ - New-Item -ItemType Directory -Path $buildFolderPath | Out-Null - Write-Host "build folder created." +if (-not (Test-Path $buildFolderPath)) { + New-Item -ItemType Directory -Path $buildFolderPath | Out-Null + Write-Host "build folder created." } -cmake -G "Visual Studio 17 2022" -A x64 -S . -B ./build/ +# cmake -G "Visual Studio 17 2022" -A x64 -S . -B ./build/ +cmake --preset=default -if ($LASTEXITCODE -eq 0) -{ - cmake --build ./build/ --config DEBUG +if ($LASTEXITCODE -eq 0) { + cmake --build build } diff --git a/scripts/llaunch.ps1 b/scripts/llaunch.ps1 index 4c23a29..921f36a 100644 --- a/scripts/llaunch.ps1 +++ b/scripts/llaunch.ps1 @@ -31,10 +31,11 @@ if (-not (Test-Path $buildFolderPath)) { Write-Host "build folder created." } -cmake -G "Visual Studio 17 2022" -A x64 -S . -B ./build/ +# cmake -G -A x64 -DCMAKE_TOOLCHAIN_FILE=C:/Users/SonnyCalcr/scoop/apps/vcpkg/current/scripts/buildsystems/vcpkg.cmake -S . -B ./build/ +cmake --preset=default if ($LASTEXITCODE -eq 0) { - cmake --build ./build/ --config DEBUG + cmake --build build if ($LASTEXITCODE -eq 0) { $exePath = getExePathFromCMakeLists Write-Host "start running as follows..." @@ -42,4 +43,3 @@ if ($LASTEXITCODE -eq 0) { Invoke-Expression $exePath } } - diff --git a/scripts/prepare_env.py b/scripts/prepare_env.py new file mode 100644 index 0000000..567322b --- /dev/null +++ b/scripts/prepare_env.py @@ -0,0 +1,96 @@ +import os + + +def normpath(path): + return path.replace("\\", "/") + + +local_app_data_dir = os.environ.get("LOCALAPPDATA") +home_dir = os.path.expanduser("~") + +cur_file_path = os.path.dirname(os.path.abspath(__file__)) +project_root_path = os.path.dirname(cur_file_path) + +user_home = os.path.expanduser("~") + +googlepinyinime_rev_root_path = normpath(project_root_path) +googlepinyinime_rev_src_path = normpath( + os.path.join(googlepinyinime_rev_root_path, "src") +) +vcpkg_include_path = normpath( + os.path.join( + googlepinyinime_rev_root_path, + "build", + "vcpkg_installed", + "x64-windows", + "include", + ) +) +webview2_path = normpath( + os.path.join( + user_home, + ".nuget", + "packages", + "microsoft.web.webview2", + "1.0.3240.44", + "build", + "native", + "include", + ) +) +wim_path = normpath( + os.path.join( + user_home, + ".nuget", + "packages", + "microsoft.windows.implementationlibrary", + "1.0.240803.1", + "include", + ) +) +boost_path = normpath(os.path.join(user_home, "scoop", "apps", "boost", "current")) + +# +# project_root/.clangd +# +dot_clangd_file = os.path.join( + googlepinyinime_rev_root_path, "scripts", "config_files", ".clangd" +) +dot_clangd_output_file = os.path.join(googlepinyinime_rev_root_path, ".clangd") +with open(dot_clangd_file, "r", encoding="utf-8") as f: + lines = f.readlines() +lines[5] = ( + f' "-I{googlepinyinime_rev_root_path}/build/vcpkg_installed/x64-windows/include",\n' +) +with open(dot_clangd_output_file, "w", encoding="utf-8") as f: + f.writelines(lines) + +# +# project_root/tests/CMakeLists.txt +# +CMakeLists_file = os.path.join( + googlepinyinime_rev_root_path, "scripts", "config_files", "CMakeLists.txt" +) +CMakeLists_output_file = os.path.join(googlepinyinime_rev_root_path, "CMakeLists.txt") +with open(CMakeLists_file, "r", encoding="utf-8") as f: + lines = f.readlines() +with open(CMakeLists_output_file, "w", encoding="utf-8") as f: + f.writelines(lines) + +# +# CMakePresets.json +# +CMakePresets_file = os.path.join( + googlepinyinime_rev_root_path, "scripts", "config_files", "CMakePresets.json" +) +CMakePresets_file_output_file = os.path.join( + googlepinyinime_rev_root_path, "CMakePresets.json" +) +with open(CMakePresets_file, "r", encoding="utf-8") as f: + lines = f.readlines() +lines[8] = f' "VCPKG_ROOT": "{normpath(user_home)}/scoop/apps/vcpkg/current/"\n' +lines[11] = ( + f' "CMAKE_TOOLCHAIN_FILE": "{normpath(user_home)}/scoop/apps/vcpkg/current/scripts/buildsystems/vcpkg.cmake",\n' +) +with open(CMakePresets_file_output_file, "w", encoding="utf-8") as f: + f.writelines(lines) diff --git a/tests/main.cpp b/tests/main.cpp index c5d225b..821c728 100644 --- a/tests/main.cpp +++ b/tests/main.cpp @@ -1,16 +1,22 @@ #include "../src/include/pinyinime.h" -#include #include #include -#include #include #include #include +#include +#include std::string fromUtf16(const ime_pinyin::char16 *buf, size_t len) { - std::u16string utf16Str(reinterpret_cast(buf), len); - std::wstring_convert, char16_t> convert; - return convert.to_bytes(utf16Str); + // 先将输入的 UTF-16 buffer 转成 std::u16string（等价于 char16_t） + const char16_t *utf16Data = reinterpret_cast(buf); + std::u16string utf16Str(utf16Data, len); + + std::string utf8Result; + // 使用 utfcpp 提供的 utf16to8 转换迭代器，将 UTF-16 转为 UTF-8 + utf8::utf16to8(utf16Str.begin(), utf16Str.end(), std::back_inserter(utf8Result)); + + return utf8Result; } void test_pinyin_search_and_segment(const std::string &user_pinyin) { @@ -88,6 +94,9 @@ void test_pinyin_search_when_retriving_first_element(const std::string &user_pin } int main() { +#ifdef _WIN32 + SetConsoleOutputCP(CP_UTF8); +#endif ime_pinyin::im_set_max_lens(64, 32); if (!ime_pinyin::im_open_decoder("./data/dict_pinyin.dat", "./data/user_dict.dat")) { std::cout << "fany bug.\n"; @@ -107,5 +116,6 @@ int main() { test_pinyin_search_and_segment("ni'shuo'ni'ma'ne"); test_pinyin_search_when_retriving_first_element("qunimadegouridequsibawonengzenmeban"); test_pinyin_search_when_retriving_first_element("zh'ta'ma'an'jing"); + test_pinyin_search_when_retriving_first_element("ni'shuo'ne'ma'de"); return 0; } \ No newline at end of file diff --git a/vcpkg-configuration.json b/vcpkg-configuration.json new file mode 100644 index 0000000..18f3e1c --- /dev/null +++ b/vcpkg-configuration.json @@ -0,0 +1,14 @@ +{ + "default-registry": { + "kind": "git", + "baseline": "b02e341c927f16d991edbd915d8ea43eac52096c", + "repository": "https://github.com/microsoft/vcpkg" + }, + "registries": [ + { + "kind": "artifact", + "location": "https://github.com/microsoft/vcpkg-ce-catalog/archive/refs/heads/main.zip", + "name": "microsoft" + } + ] +} diff --git a/vcpkg.json b/vcpkg.json new file mode 100644 index 0000000..d2b69cb --- /dev/null +++ b/vcpkg.json @@ -0,0 +1,5 @@ +{ + "dependencies": [ + "utfcpp" + ] +}