You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
345 lines
6.4 KiB
345 lines
6.4 KiB
# Copyright 2023-2024 Gentoo Authors
|
|
# Distributed under the terms of the GNU General Public License v2
|
|
|
|
# Autogenerated by pycargoebuild 0.10
|
|
|
|
EAPI=8
|
|
|
|
DISTUTILS_USE_PEP517=maturin
|
|
PYTHON_COMPAT=( python3_{10..12} )
|
|
DISTUTILS_EXT=1
|
|
DISTUTILS_SINGLE_IMPL=1
|
|
|
|
CRATES="
|
|
adler@1.0.2
|
|
aho-corasick@1.1.2
|
|
aho-corasick@1.1.3
|
|
anes@0.1.6
|
|
anstream@0.6.11
|
|
anstream@0.6.13
|
|
anstyle-parse@0.2.3
|
|
anstyle-query@1.0.2
|
|
anstyle-wincon@3.0.2
|
|
anstyle@1.0.6
|
|
assert_approx_eq@1.1.0
|
|
autocfg@1.1.0
|
|
autocfg@1.2.0
|
|
base64@0.13.1
|
|
base64@0.21.7
|
|
bit-set@0.5.3
|
|
bit-vec@0.6.3
|
|
bitflags@1.3.2
|
|
bitflags@2.4.2
|
|
bitflags@2.5.0
|
|
bumpalo@3.15.4
|
|
cast@0.3.0
|
|
cc@1.0.83
|
|
cc@1.0.90
|
|
cfg-if@1.0.0
|
|
ciborium-io@0.2.2
|
|
ciborium-ll@0.2.2
|
|
ciborium@0.2.2
|
|
clap@4.5.0
|
|
clap@4.5.4
|
|
clap_builder@4.5.0
|
|
clap_builder@4.5.2
|
|
clap_derive@4.5.0
|
|
clap_derive@4.5.4
|
|
clap_lex@0.7.0
|
|
colorchoice@1.0.0
|
|
console@0.15.8
|
|
core-foundation-sys@0.8.6
|
|
core-foundation@0.9.4
|
|
crc32fast@1.4.0
|
|
criterion-plot@0.5.0
|
|
criterion@0.5.1
|
|
crossbeam-deque@0.8.5
|
|
crossbeam-epoch@0.9.18
|
|
crossbeam-utils@0.8.19
|
|
crunchy@0.2.2
|
|
darling@0.14.4
|
|
darling_core@0.14.4
|
|
darling_macro@0.14.4
|
|
derive_builder@0.12.0
|
|
derive_builder_core@0.12.0
|
|
derive_builder_macro@0.12.0
|
|
dirs-sys@0.4.1
|
|
dirs@5.0.1
|
|
either@1.10.0
|
|
encode_unicode@0.3.6
|
|
env_logger@0.10.2
|
|
errno@0.3.8
|
|
esaxx-rs@0.1.10
|
|
fancy-regex@0.13.0
|
|
fastrand@2.0.1
|
|
fastrand@2.0.2
|
|
flate2@1.0.28
|
|
fnv@1.0.7
|
|
foreign-types-shared@0.1.1
|
|
foreign-types@0.3.2
|
|
form_urlencoded@1.2.1
|
|
getrandom@0.2.12
|
|
half@2.4.0
|
|
heck@0.4.1
|
|
heck@0.5.0
|
|
hermit-abi@0.3.5
|
|
hermit-abi@0.3.9
|
|
hf-hub@0.3.2
|
|
humantime@2.1.0
|
|
ident_case@1.0.1
|
|
idna@0.5.0
|
|
indicatif@0.17.8
|
|
indoc@2.0.4
|
|
indoc@2.0.5
|
|
instant@0.1.12
|
|
is-terminal@0.4.12
|
|
itertools@0.10.5
|
|
itertools@0.11.0
|
|
itertools@0.12.1
|
|
itoa@1.0.10
|
|
itoa@1.0.11
|
|
js-sys@0.3.69
|
|
lazy_static@1.4.0
|
|
libc@0.2.153
|
|
libredox@0.1.3
|
|
linux-raw-sys@0.4.13
|
|
lock_api@0.4.11
|
|
log@0.4.20
|
|
log@0.4.21
|
|
macro_rules_attribute-proc_macro@0.2.0
|
|
macro_rules_attribute@0.2.0
|
|
matrixmultiply@0.3.8
|
|
memchr@2.7.1
|
|
memchr@2.7.2
|
|
memoffset@0.9.0
|
|
memoffset@0.9.1
|
|
minimal-lexical@0.2.1
|
|
miniz_oxide@0.7.2
|
|
monostate-impl@0.1.11
|
|
monostate@0.1.11
|
|
native-tls@0.2.11
|
|
ndarray@0.15.6
|
|
nom@7.1.3
|
|
num-complex@0.4.5
|
|
num-integer@0.1.46
|
|
num-traits@0.2.18
|
|
number_prefix@0.4.0
|
|
numpy@0.20.0
|
|
once_cell@1.19.0
|
|
onig@6.4.0
|
|
onig_sys@69.8.1
|
|
oorandom@11.1.3
|
|
openssl-macros@0.1.1
|
|
openssl-probe@0.1.5
|
|
openssl-sys@0.9.102
|
|
openssl@0.10.64
|
|
option-ext@0.2.0
|
|
parking_lot@0.12.1
|
|
parking_lot_core@0.9.9
|
|
paste@1.0.14
|
|
percent-encoding@2.3.1
|
|
pkg-config@0.3.29
|
|
pkg-config@0.3.30
|
|
plotters-backend@0.3.5
|
|
plotters-svg@0.3.5
|
|
plotters@0.3.5
|
|
portable-atomic@1.6.0
|
|
ppv-lite86@0.2.17
|
|
proc-macro2@1.0.78
|
|
proc-macro2@1.0.79
|
|
pyo3-build-config@0.20.2
|
|
pyo3-build-config@0.20.3
|
|
pyo3-ffi@0.20.2
|
|
pyo3-ffi@0.20.3
|
|
pyo3-macros-backend@0.20.2
|
|
pyo3-macros-backend@0.20.3
|
|
pyo3-macros@0.20.2
|
|
pyo3-macros@0.20.3
|
|
pyo3@0.20.2
|
|
pyo3@0.20.3
|
|
quote@1.0.35
|
|
rand@0.8.5
|
|
rand_chacha@0.3.1
|
|
rand_core@0.6.4
|
|
rawpointer@0.2.1
|
|
rayon-cond@0.3.0
|
|
rayon-core@1.12.1
|
|
rayon@1.8.1
|
|
rayon@1.10.0
|
|
redox_syscall@0.4.1
|
|
regex-automata@0.4.5
|
|
regex-syntax@0.8.2
|
|
redox_users@0.4.5
|
|
regex-automata@0.4.6
|
|
regex-syntax@0.8.3
|
|
regex@1.10.3
|
|
regex@1.10.4
|
|
ring@0.17.8
|
|
rustc-hash@1.1.0
|
|
rustix@0.38.31
|
|
rustix@0.38.32
|
|
rustls-pki-types@1.4.1
|
|
rustls-webpki@0.102.2
|
|
rustls@0.22.3
|
|
ryu@1.0.16
|
|
ryu@1.0.17
|
|
scopeguard@1.2.0
|
|
same-file@1.0.6
|
|
schannel@0.1.23
|
|
scopeguard@1.2.0
|
|
security-framework-sys@2.10.0
|
|
security-framework@2.10.0
|
|
serde@1.0.196
|
|
serde@1.0.197
|
|
serde_derive@1.0.196
|
|
serde_derive@1.0.197
|
|
serde_json@1.0.113
|
|
serde_json@1.0.115
|
|
smallvec@1.13.1
|
|
smallvec@1.13.2
|
|
spin@0.9.8
|
|
spm_precompiled@0.1.4
|
|
strsim@0.10.0
|
|
strsim@0.11.0
|
|
strsim@0.11.1
|
|
subtle@2.5.0
|
|
syn@1.0.109
|
|
syn@2.0.48
|
|
syn@2.0.58
|
|
target-lexicon@0.12.13
|
|
target-lexicon@0.12.14
|
|
tempfile@3.10.0
|
|
tempfile@3.10.1
|
|
termcolor@1.4.1
|
|
thiserror-impl@1.0.56
|
|
thiserror-impl@1.0.58
|
|
thiserror@1.0.56
|
|
thiserror@1.0.58
|
|
tinytemplate@1.2.1
|
|
tinyvec@1.6.0
|
|
tinyvec_macros@0.1.1
|
|
unicode-bidi@0.3.15
|
|
unicode-ident@1.0.12
|
|
unicode-normalization-alignments@0.1.12
|
|
unicode-normalization@0.1.23
|
|
unicode-segmentation@1.11.0
|
|
unicode-width@0.1.11
|
|
unicode_categories@0.1.1
|
|
unindent@0.2.3
|
|
untrusted@0.9.0
|
|
ureq@2.9.6
|
|
url@2.5.0
|
|
utf8parse@0.2.1
|
|
vcpkg@0.2.15
|
|
walkdir@2.5.0
|
|
wasi@0.11.0+wasi-snapshot-preview1
|
|
wasm-bindgen-backend@0.2.92
|
|
wasm-bindgen-macro-support@0.2.92
|
|
wasm-bindgen-macro@0.2.92
|
|
wasm-bindgen-shared@0.2.92
|
|
wasm-bindgen@0.2.92
|
|
web-sys@0.3.69
|
|
webpki-roots@0.26.1
|
|
winapi-i686-pc-windows-gnu@0.4.0
|
|
winapi-util@0.1.6
|
|
winapi-x86_64-pc-windows-gnu@0.4.0
|
|
winapi@0.3.9
|
|
windows-sys@0.48.0
|
|
windows-sys@0.52.0
|
|
windows-targets@0.48.5
|
|
windows-targets@0.52.0
|
|
windows-targets@0.52.4
|
|
windows_aarch64_gnullvm@0.48.5
|
|
windows_aarch64_gnullvm@0.52.0
|
|
windows_aarch64_gnullvm@0.52.4
|
|
windows_aarch64_msvc@0.48.5
|
|
windows_aarch64_msvc@0.52.0
|
|
windows_aarch64_msvc@0.52.4
|
|
windows_i686_gnu@0.48.5
|
|
windows_i686_gnu@0.52.0
|
|
windows_i686_gnu@0.52.4
|
|
windows_i686_msvc@0.48.5
|
|
windows_i686_msvc@0.52.0
|
|
windows_i686_msvc@0.52.4
|
|
windows_x86_64_gnu@0.48.5
|
|
windows_x86_64_gnu@0.52.0
|
|
windows_x86_64_gnu@0.52.4
|
|
windows_x86_64_gnullvm@0.48.5
|
|
windows_x86_64_gnullvm@0.52.0
|
|
windows_x86_64_gnullvm@0.52.4
|
|
windows_x86_64_msvc@0.48.5
|
|
windows_x86_64_msvc@0.52.0
|
|
windows_x86_64_msvc@0.52.4
|
|
zeroize@1.7.0
|
|
"
|
|
|
|
inherit cargo distutils-r1
|
|
|
|
DESCRIPTION="Implementation of today's most used tokenizers"
|
|
HOMEPAGE="https://github.com/huggingface/tokenizers"
|
|
SRC_URI="
|
|
https://github.com/huggingface/${PN}/archive/refs/tags/v${PV}.tar.gz
|
|
-> ${P}.gh.tar.gz
|
|
${CARGO_CRATE_URIS}
|
|
"
|
|
|
|
LICENSE="Apache-2.0"
|
|
# Dependent crate licenses
|
|
LICENSE+="
|
|
Apache-2.0 Apache-2.0-with-LLVM-exceptions BSD-2 BSD ISC MIT MPL-2.0
|
|
Unicode-DFS-2016
|
|
"
|
|
SLOT="0"
|
|
KEYWORDS="~amd64"
|
|
|
|
BDEPEND="
|
|
test? ( sci-libs/datasets[${PYTHON_SINGLE_USEDEP}] )
|
|
$(python_gen_cond_dep '
|
|
dev-python/setuptools-rust[${PYTHON_USEDEP}]
|
|
')
|
|
"
|
|
|
|
distutils_enable_tests pytest
|
|
|
|
QA_FLAGS_IGNORED="/usr/bin/cli .*/site-packages/tokenizers/.*so"
|
|
|
|
src_unpack() {
|
|
cargo_src_unpack
|
|
}
|
|
|
|
src_prepare() {
|
|
default
|
|
cd bindings/python
|
|
eapply "${FILESDIR}"/${P}-test.patch
|
|
distutils-r1_src_prepare
|
|
}
|
|
|
|
src_configure() {
|
|
cd tokenizers
|
|
cargo_src_configure
|
|
cd ../bindings/python
|
|
distutils-r1_src_configure
|
|
}
|
|
|
|
src_compile() {
|
|
cd tokenizers
|
|
cargo_src_compile
|
|
cd ../bindings/python
|
|
distutils-r1_src_compile
|
|
}
|
|
|
|
src_test() {
|
|
cd tokenizers
|
|
# Tests do not work
|
|
#cargo_src_test
|
|
cd ../bindings/python
|
|
distutils-r1_src_test
|
|
}
|
|
|
|
src_install() {
|
|
cd tokenizers
|
|
cargo_src_install
|
|
cd ../bindings/python
|
|
distutils-r1_src_install
|
|
}
|