"tokenize" meaning in Englisch

{
  "categories": [
    {
      "kind": "other",
      "name": "Anagramm sortiert (Englisch)",
      "parents": [],
      "source": "w"
    },
    {
      "kind": "other",
      "name": "Englisch",
      "parents": [],
      "source": "w"
    },
    {
      "kind": "other",
      "name": "Rückläufige Wörterliste (Englisch)",
      "parents": [],
      "source": "w"
    },
    {
      "kind": "other",
      "name": "Verb (Englisch)",
      "parents": [],
      "source": "w"
    },
    {
      "kind": "other",
      "name": "Wartung fehlende Lautschrift (Englisch)",
      "parents": [],
      "source": "w"
    },
    {
      "kind": "other",
      "name": "Siehe auch",
      "orig": "siehe auch",
      "parents": [],
      "source": "w"
    }
  ],
  "forms": [
    {
      "form": "tokenizes",
      "pronouns": [
        "he",
        "she",
        "it"
      ],
      "tags": [
        "present"
      ]
    },
    {
      "form": "tokenized",
      "tags": [
        "past"
      ]
    },
    {
      "form": "tokenizing",
      "tags": [
        "present",
        "participle"
      ]
    },
    {
      "form": "tokenized",
      "tags": [
        "past",
        "participle"
      ]
    }
  ],
  "hyphenations": [
    {
      "parts": [
        "to",
        "ke",
        "nize"
      ]
    }
  ],
  "lang": "Englisch",
  "lang_code": "en",
  "pos": "verb",
  "pos_title": "Verb",
  "senses": [
    {
      "examples": [
        {
          "bold_text_offsets": [
            [
              32,
              40
            ]
          ],
          "ref": "Pierre M. Nugues: Language Processing with Perl and Prolog. Springer, 2014, Seite 127 (Google Books)",
          "text": "„The previous program failed to tokenize the puctuation.“"
        },
        {
          "bold_text_offsets": [
            [
              23,
              31
            ]
          ],
          "ref": "Eric Rochester: Clojure Data Analysis Cookbook. PACKT Publishing, 2015, Seite 64 (Google Books)",
          "text": "„We need a function to tokenize a string into words.“"
        },
        {
          "bold_text_offsets": [
            [
              14,
              23
            ]
          ],
          "ref": "David Smiley,Eric Pugh,Kranti Parisa,Matt Mitchell: Apache Solr Enterprise Search Server. PACKT Publishing, 2015, Seite 59 (Google Books)",
          "text": "„This example tokenizes a semi-colon separated list.“"
        }
      ],
      "glosses": [
        "zerlegen eines Textes für die Weiterverarbeitung"
      ],
      "id": "de-tokenize-en-verb-DXd4qcWp",
      "raw_tags": [
        "Computerlinguistik"
      ],
      "sense_index": "1"
    }
  ],
  "sounds": [
    {
      "ipa": "[…]"
    }
  ],
  "translations": [
    {
      "lang": "Deutsch",
      "lang_code": "de",
      "sense": "Computerlinguistik: zerlegen eines Textes für die Weiterverarbeitung",
      "sense_index": "1",
      "word": "tokenisieren"
    }
  ],
  "word": "tokenize"
}

{
  "categories": [
    "Anagramm sortiert (Englisch)",
    "Englisch",
    "Rückläufige Wörterliste (Englisch)",
    "Verb (Englisch)",
    "Wartung fehlende Lautschrift (Englisch)",
    "siehe auch"
  ],
  "forms": [
    {
      "form": "tokenizes",
      "pronouns": [
        "he",
        "she",
        "it"
      ],
      "tags": [
        "present"
      ]
    },
    {
      "form": "tokenized",
      "tags": [
        "past"
      ]
    },
    {
      "form": "tokenizing",
      "tags": [
        "present",
        "participle"
      ]
    },
    {
      "form": "tokenized",
      "tags": [
        "past",
        "participle"
      ]
    }
  ],
  "hyphenations": [
    {
      "parts": [
        "to",
        "ke",
        "nize"
      ]
    }
  ],
  "lang": "Englisch",
  "lang_code": "en",
  "pos": "verb",
  "pos_title": "Verb",
  "senses": [
    {
      "examples": [
        {
          "bold_text_offsets": [
            [
              32,
              40
            ]
          ],
          "ref": "Pierre M. Nugues: Language Processing with Perl and Prolog. Springer, 2014, Seite 127 (Google Books)",
          "text": "„The previous program failed to tokenize the puctuation.“"
        },
        {
          "bold_text_offsets": [
            [
              23,
              31
            ]
          ],
          "ref": "Eric Rochester: Clojure Data Analysis Cookbook. PACKT Publishing, 2015, Seite 64 (Google Books)",
          "text": "„We need a function to tokenize a string into words.“"
        },
        {
          "bold_text_offsets": [
            [
              14,
              23
            ]
          ],
          "ref": "David Smiley,Eric Pugh,Kranti Parisa,Matt Mitchell: Apache Solr Enterprise Search Server. PACKT Publishing, 2015, Seite 59 (Google Books)",
          "text": "„This example tokenizes a semi-colon separated list.“"
        }
      ],
      "glosses": [
        "zerlegen eines Textes für die Weiterverarbeitung"
      ],
      "raw_tags": [
        "Computerlinguistik"
      ],
      "sense_index": "1"
    }
  ],
  "sounds": [
    {
      "ipa": "[…]"
    }
  ],
  "translations": [
    {
      "lang": "Deutsch",
      "lang_code": "de",
      "sense": "Computerlinguistik: zerlegen eines Textes für die Weiterverarbeitung",
      "sense_index": "1",
      "word": "tokenisieren"
    }
  ],
  "word": "tokenize"
}