<?php
require_once ('tokenizer.php');
/******* TOKENIZER EXAMPLES ********/
// Simple example
/*
[0]=> "Hello"
[1]=> " "
[2]=> "world!"
*/
$tokenizer = tokenizer::registerDefaultTokenizer();
var_dump ($tokenizer->tokenize ('Hello world!'));
// Example with markers
/*
[0]=> "Please"
[1]=> " "
[2]=> "not"
[3]=> " "
[4]=> ""cut quotation points"!"
*/
var_dump ($tokenizer->tokenize ('Please not "cut quotation points"!'));
// Example with escaping char
/*
[0]=> "Mark"
[1]=> " "
[2]=> "said:"
[3]=> " "
[4]=> "'where is Janet\'s jacket'?"
*/
var_dump ($tokenizer->tokenize ("Mark said: 'where is Janet\'s jacket'?"));
?>
|
Download
