8000 fasttext, feat: ad fasttext module. · fibjs/fibjs@de5ad3a · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Commit de5ad3a

Browse files
xicilionxicilion
committed
fasttext, feat: ad fasttext module.
1 parent aa7333f commit de5ad3a

File tree

12 files changed

+405
-2
lines changed

12 files changed

+405
-2
lines changed

‎fibjs/include/FTModel.h

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* FTModel.cpp
3+
*
4+
* Created on: Sep 20, 2023
5+
* Author: lion
6+
*/
7+
8+
#pragma once
9+
10+
#include "ifs/FTModel.h"
11+
#include "fasttext/include/fasttext.h"
12+
13+
namespace fibjs {
14+
15+
class FTModel : public FTModel_base {
16+
public:
17+
// FTModel_base
18+
virtual result_t predict(exlib::string text, int32_t k, obj_ptr<NArray>& retVal, AsyncEvent* ac);
19+
virtual result_t predict(v8::Local<v8::Array> texts, int32_t k, obj_ptr<NArray>& retVal, AsyncEvent* ac);
20+
21+
public:
22+
result_t loadModel(exlib::string path);
23+
24+
private:
25+
fasttext::FastText fastText_;
26+
};
27+
28+
} // namespace fibjs

‎fibjs/include/ifs/FTModel.h

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
/***************************************************************************
2+
* *
3+
* This file was automatically generated using idlc.js *
4+
* PLEASE DO NOT EDIT!!!! *
5+
* *
6+
***************************************************************************/
7+
8+
#pragma once
9+
10+
/**
11+
@author Leo Hoo <lion@9465.net>
12+
*/
13+
14+
#include "../object.h"
15+
16+
namespace fibjs {
17+
18+
class FTModel_base : public object_base {
19+
DECLARE_CLASS(FTModel_base);
20+
21+
public:
22+
// FTModel_base
23+
virtual result_t predict(exlib::string text, int32_t k, obj_ptr<NArray>& retVal, AsyncEvent* ac) = 0;
24+
virtual result_t predict(v8::Local<v8::Array> texts, int32_t k, obj_ptr<NArray>& retVal, AsyncEvent* ac) = 0;
25+
26+
public:
27+
static void s__new(const v8::FunctionCallbackInfo<v8::Value>& args)
28+
{
29+
CONSTRUCT_INIT();
30+
31+
isolate->m_isolate->ThrowException(
32+
isolate->NewString("not a constructor"));
33+
}
34+
35+
public:
36+
static void s_predict(const v8::FunctionCallbackInfo<v8::Value>& args);
37+
38+
public:
39+
ASYNC_MEMBERVALUE3(FTModel_base, predict, exlib::string, int32_t, obj_ptr<NArray>);
40+
ASYNC_MEMBERVALUE3(FTModel_base, predict, v8::Local<v8::Array>, int32_t, obj_ptr<NArray>);
41+
};
42+
}
43+
44+
namespace fibjs {
45+
inline ClassInfo& FTModel_base::class_info()
46+
{
47+
static ClassData::ClassMethod s_method[] = {
48+
{ "predict", s_predict, false, true },
49+
{ "predictSync", s_predict, false, false }
50 F438 +
};
51+
52+
static ClassData s_cd = {
53+
"FTModel", false, s__new, NULL,
54+
ARRAYSIZE(s_method), s_method, 0, NULL, 0, NULL, 0, NULL, NULL, NULL,
55+
&object_base::class_info(),
56+
true
57+
};
58+
59+
static ClassInfo s_ci(s_cd);
60+
return s_ci;
61+
}
62+
63+
inline void FTModel_base::s_predict(const v8::FunctionCallbackInfo<v8::Value>& args)
64+
{
65+
obj_ptr<NArray> vr;
66+
67+
ASYNC_METHOD_INSTANCE(FTModel_base);
68+
METHOD_ENTER();
69+
70+
ASYNC_METHOD_OVER(2, 1);
71+
72+
ARG(exlib::string, 0);
73+
OPT_ARG(int32_t, 1, 3);
74+
75+
if (!cb.IsEmpty())
76+
hr = pInst->acb_predict(v0, v1, cb, args);
77+
else
78+
hr = pInst->ac_predict(v0, v1, vr);
79+
80+
ASYNC_METHOD_OVER(2, 1);
81+
82+
ARG(v8::Local<v8::Array>, 0);
83+
OPT_ARG(int32_t, 1, 3);
84+
85+
if (!cb.IsEmpty())
86+
hr = pInst->acb_predict(v0, v1, cb, args);
87+
else
88+
hr = pInst->ac_predict(v0, v1, vr);
89+
90+
METHOD_RETURN();
91+
}
92+
}

‎fibjs/include/ifs/fasttext.h

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
/***************************************************************************
2+
* *
3+
* This file was automatically generated using idlc.js *
4+
* PLEASE DO NOT EDIT!!!! *
5+
* *
6+
***************************************************************************/
7+
8+
#pragma once
9+
10+
/**
11+
@author Leo Hoo <lion@9465.net>
12+
*/
13+
14+
#include "../object.h"
15+
16+
namespace fibjs {
17+
18+
class FTModel_base;
19+
20+
class fasttext_base : public object_base {
21+
DECLARE_CLASS(fasttext_base);
22+
23+
public:
24+
// fasttext_base
25+
static result_t loadModel(exlib::string path, obj_ptr<FTModel_base>& retVal, AsyncEvent* ac);
26+
27+
public:
28+
static void s__new(const v8::FunctionCallbackInfo<v8::Value>& args)
29+
{
30+
CONSTRUCT_INIT();
31+
32+
isolate->m_isolate->ThrowException(
33+
isolate->NewString("not a constructor"));
34+
}
35+
36+
public:
37+
static void s_static_loadModel(const v8::FunctionCallbackInfo<v8::Value>& args);
38+
39+
public:
40+
ASYNC_STATICVALUE2(fasttext_base, loadModel, exlib::string, obj_ptr<FTModel_base>);
41+
};
42+
}
43+
44+
#include "ifs/FTModel.h"
45+
46+
namespace fibjs {
47+
inline ClassInfo& fasttext_base::class_info()
48+
{
49+
static ClassData::ClassMethod s_method[] = {
50+
{ "loadModel", s_static_loadModel, true, true },
51+
{ "loadModelSync", s_static_loadModel, true, false }
52+
};
53+
54+
static ClassData s_cd = {
55+
"fasttext", true, s__new, NULL,
56+
ARRAYSIZE(s_method), s_method, 0, NULL, 0, NULL, 0, NULL, NULL, NULL,
57+
&object_base::class_info(),
58+
true
59+
};
60+
61+
static ClassInfo s_ci(s_cd);
62+
return s_ci;
63+
}
64+
65+
inline void fasttext_base::s_static_loadModel(const v8::FunctionCallbackInfo<v8::Value>& args)
66+
{
67+
obj_ptr<FTModel_base> vr;
68+
69+
METHOD_ENTER();
70+
71+
ASYNC_METHOD_OVER(1, 1);
72+
73+
ARG(exlib::string, 0);
74+
75+
if (!cb.IsEmpty())
76+
hr = acb_loadModel(v0, cb, args);
77+
else
78+
hr = ac_loadModel(v0, vr);
79+
80+
METHOD_RETURN();
81+
}
82+
}

‎fibjs/program/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ add_library(js SHARED src/so.cpp)
2525

2626
include_directories("${PROJECT_SOURCE_DIR}/include" "${PROJECT_SOURCE_DIR}/../include" "${PROJECT_SOURCE_DIR}/../../vender" "${PROJECT_SOURCE_DIR}/../../vender/v8" "${PROJECT_SOURCE_DIR}/../../vender/v8/include" "${PROJECT_SOURCE_DIR}/../../vender/mbedtls" "${PROJECT_SOURCE_DIR}/../../vender/zlib/include" "${CMAKE_CURRENT_BINARY_DIR}")
2727

28-
set(libs fibjs jemalloc expat gumbo gd tiff jpeg png webp zlib leveldb snappy msgpack pcre sqlite umysql uuid exif mbedtls curve25519 blst blake2 fips202 v8 unicode unzip uv exlib)
28+
set(libs fibjs jemalloc expat gumbo gd tiff jpeg png webp zlib leveldb snappy msgpack pcre sqlite umysql uuid exif mbedtls curve25519 blst blake2 fips202 v8 unicode unzip uv fasttext exlib)
2929

3030
if(${CMAKE_HOST_SYSTEM_NAME} STREQUAL "Windows")
3131
list(APPEND libs winiconv)

‎fibjs/src/base/modules.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ void importBuiltinModule()
2525
IMPORT_MODULE(dns);
2626
IMPORT_MODULE(encoding);
2727
IMPORT_MODULE(events);
28+
IMPORT_MODULE(fasttext);
2829
IMPORT_MODULE(fs);
2930
IMPORT_MODULE(gd);
3031
IMPORT_MODULE(gui);

‎fibjs/src/fasttext/fasttext.cpp

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
/*
2+
* fasttext.cpp
3+
*
4+
* Created on: Sep 20, 2023
5+
* Author: lion
6+
*/
7+
8+
#include "ifs/fasttext.h"
9+
#include "FTModel.h"
10+
#include <iostream>
11+
#include <fstream>
12+
#include <sstream>
13+
14+
namespace fibjs {
15+
16+
DECLARE_MODULE(fasttext);
17+
18+
result_t fasttext_base::loadModel(exlib::string path, obj_ptr<FTModel_base>& retVal, AsyncEvent* ac)
19+
{
20+
if (ac->isSync())
21+
return CHECK_ERROR(CALL_E_NOSYNC);
22+
23+
obj_ptr<FTModel> model = new FTModel();
24+
25+
result_t hr = model->loadModel(path);
26+
if (hr < 0)
27+
return hr;
28+
29+
retVal = model;
30+
31+
return 0;
32+
}
33+
34+
result_t FTModel::loadModel(exlib::string path)
35+
{
36+
try {
37+
fastText_.loadModel(path);
38+
} catch (const std::exception& e) {
39+
return CHECK_ERROR(Runtime::setError("fasttext: " + exlib::string(e.what())));
40+
}
41+
42+
return 0;
43+
}
44+
45+
result_t FTModel::predict(exlib::string text, int32_t k, obj_ptr<NArray>& retVal, AsyncEvent* ac)
46+
{
47+
if (ac->isSync())
48+
return CHECK_ERROR(CALL_E_NOSYNC);
49+
50+
std::vector<std::pair<fasttext::real, std::string>> predictions;
51+
std::istringstream in;
52+
in.str(text);
53+
54+
try {
55+
fastText_.predictLine(in, predictions, k, 0);
56+
} catch (const std::exception& e) {
57+
return CHECK_ERROR(Runtime::setError("fasttext: " + exlib::string(e.what())));
58+
}
59+
60+
retVal = new NArray();
61+
for (auto it = predictions.cbegin(); it != predictions.cend(); it++) {
62+
obj_ptr<NObject> obj = new NObject();
63+
64+
obj->add("label", exlib::string(it->second));
65+
obj->add("value", it->first);
66+
67+
retVal->append(obj);
68+
}
69+
70+
return 0;
71+
}
72+
73+
result_t FTModel::predict(v8::Local<v8::Array> texts, int32_t k, obj_ptr<NArray>& retVal, AsyncEvent* ac)
74+
{
75+
if (ac->isSync()) {
76+
ac->m_ctx.resize(1);
77+
78+
obj_ptr<NArray> _texts = new NArray();
79+
result_t hr = _texts->append_array<exlib::string>(texts);
80+
if (hr < 0)
81+
return hr;
82+
83+
ac->m_ctx[0] = _texts;
84+
85+
return CHECK_ERROR(CALL_E_NOSYNC);
86+
}
87+
88+
result_t hr;
89+
obj_ptr<NArray> _texts = (NArray*)ac->m_ctx[0].object();
90+
91+
retVal = new NArray();
92+
for (int i = 0; i < _texts->length(); i++) {
93+
Variant v;
94+
obj_ptr<NArray> result;
95+
_texts->_indexed_getter(i, v);
96+
97+
hr = predict(v.string(), k, result, ac);
98+
if (hr < 0)
99+
return hr;
100+
101+
retVal->append(result);
102+
}
103+
104+
return 0;
105+
}
106+
107+
} // namespace fibjs

‎idl/zh-cn/FTModel.idl

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/*!
2+
* FTModel fasttext 模型对象
3+
*
4+
*/
5+
interface FTModel : object
6+
{
7+
/*! @brief 预测文本的分类
8+
@param text 文本
9+
@param k 返回前 k 个分类
10+
@return 返回分类结果
11+
*/
12+
NArray predict(String text, Integer k = 3) async;
13+
14+
/*! @brief 预测文本的分类
15+
@param texts 文本数组
16+
@param k 返回前 k 个分类
17+
@return 返回分类结果
18+
*/
19+
NArray predict(Array texts, Integer k = 3) async;
20+
};

‎idl/zh-cn/fasttext.idl

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
/*! @brief fasttext 文本处理与分类模块
2+
3+
使用方法：
4+
```JavaScript
5+
var fasttext = require('fasttext');
6+
```
7+
*/
8+
module fasttext
9+
{
10+
/*! @brief 加载 fasttext 模型
11+
@param path 模型文件路径
12+
@return 返回加载的模型对象
13+
*/
14+
static FTModel loadModel(String path) async;
15+
};

‎npm/types/dts/_import/bridge.d.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
/// <reference path="../module/xml.d.ts" />
5252
/// <reference path="../module/constants.d.ts" />
5353
/// <reference path="../module/fs_constants.d.ts" />
54+
/// <reference path="../module/fasttext.d.ts" />
5455
declare module '@fibjs/types/bridge' {
5556
}
5657

0 commit comments

Comments
 (0)
0