See tokenization on Wiktionary
{
"etymology_templates": [
{
"args": {
"1": "en",
"2": "token",
"3": "ization"
},
"expansion": "token + -ization",
"name": "suffix"
}
],
"etymology_text": "From token + -ization.",
"forms": [
{
"form": "tokenizations",
"tags": [
"plural"
]
}
],
"head_templates": [
{
"args": {
"1": "~"
},
"expansion": "tokenization (countable and uncountable, plural tokenizations)",
"name": "en-noun"
}
],
"lang": "English",
"lang_code": "en",
"pos": "noun",
"senses": [
{
"glosses": [
"The act or process of tokenizing."
],
"id": "en-tokenization-en-noun-Haufhfbs",
"links": [
[
"tokenizing",
"tokenize"
]
],
"tags": [
"countable",
"uncountable"
]
},
{
"categories": [
{
"_dis": "40 60",
"kind": "other",
"name": "English entries with incorrect language header",
"parents": [
"Entries with incorrect language header",
"Entry maintenance"
],
"source": "w+disamb"
},
{
"_dis": "36 64",
"kind": "other",
"name": "English terms suffixed with -ization",
"parents": [],
"source": "w+disamb"
},
{
"_dis": "37 63",
"kind": "other",
"name": "Pages with 1 entry",
"parents": [],
"source": "w+disamb"
},
{
"_dis": "35 65",
"kind": "other",
"name": "Pages with entries",
"parents": [],
"source": "w+disamb"
}
],
"examples": [
{
"text": "This was an unlikely tokenization of the input string.",
"type": "example"
}
],
"glosses": [
"Something tokenized."
],
"id": "en-tokenization-en-noun-PZEdrkEY",
"tags": [
"countable",
"uncountable"
]
}
],
"wikipedia": [
"tokenization"
],
"word": "tokenization"
}
{
"categories": [
"English countable nouns",
"English entries with incorrect language header",
"English lemmas",
"English nouns",
"English terms suffixed with -ization",
"English uncountable nouns",
"Pages with 1 entry",
"Pages with entries"
],
"etymology_templates": [
{
"args": {
"1": "en",
"2": "token",
"3": "ization"
},
"expansion": "token + -ization",
"name": "suffix"
}
],
"etymology_text": "From token + -ization.",
"forms": [
{
"form": "tokenizations",
"tags": [
"plural"
]
}
],
"head_templates": [
{
"args": {
"1": "~"
},
"expansion": "tokenization (countable and uncountable, plural tokenizations)",
"name": "en-noun"
}
],
"lang": "English",
"lang_code": "en",
"pos": "noun",
"senses": [
{
"glosses": [
"The act or process of tokenizing."
],
"links": [
[
"tokenizing",
"tokenize"
]
],
"tags": [
"countable",
"uncountable"
]
},
{
"categories": [
"English terms with usage examples"
],
"examples": [
{
"text": "This was an unlikely tokenization of the input string.",
"type": "example"
}
],
"glosses": [
"Something tokenized."
],
"tags": [
"countable",
"uncountable"
]
}
],
"wikipedia": [
"tokenization"
],
"word": "tokenization"
}
Download raw JSONL data for tokenization meaning in All languages combined (1.1kB)
This page is a part of the kaikki.org machine-readable All languages combined dictionary. This dictionary is based on structured data extracted on 2024-12-21 from the enwiktionary dump dated 2024-12-04 using wiktextract (d8cb2f3 and 4e554ae). The data shown on this site has been post-processed and various details (e.g., extra categories) removed, some information disambiguated, and additional data merged from other sources. See the raw data download page for the unprocessed wiktextract data.
If you use this data in academic research, please cite Tatu Ylonen: Wiktextract: Wiktionary as Machine-Readable Structured Data, Proceedings of the 13th Conference on Language Resources and Evaluation (LREC), pp. 1317-1325, Marseille, 20-25 June 2022. Linking to the relevant page(s) under https://kaikki.org would also be greatly appreciated.