From 5d3acf74d0133643799628a4c4b62e2f69a46de9 Mon Sep 17 00:00:00 2001
From: belliash <belliash@codingworkshop.eu.org>
Date: Thu, 7 Feb 2019 18:43:58 +0100
Subject: [PATCH] Another test - tokenizer.

---
 tests/tokenizer.aer | 36 ++++++++++++++++++++++++++++++++++++
 tests/tokenizer.exp |  4 ++++
 2 files changed, 40 insertions(+)
 create mode 100644 tests/tokenizer.aer
 create mode 100644 tests/tokenizer.exp

diff --git a/tests/tokenizer.aer b/tests/tokenizer.aer
new file mode 100644
index 0000000..1393c71
--- /dev/null
+++ b/tests/tokenizer.aer
@@ -0,0 +1,36 @@
+class StringTokenizer {
+	private $token;
+	private $delim;
+
+	public function __construct(string $str, string $delim = ' ') {
+		$this->token = strtok($str, $delim);
+		$this->delim = $delim;
+	}
+
+	public function __destruct() {
+		unset($this);
+	}
+
+	public function hasMoreTokens() {
+		return ($this->token !== false);
+	}
+
+	public function nextToken() {
+		string $current = $this->token;
+		$this->token = strtok($this->delim);
+		return $current;
+	}
+
+}
+
+class Program {
+	function main() {
+		string $str = "This is:@\t\n a TEST!";
+		string $delim = " !@:\t\n";
+		object $st = new StringTokenizer($str, $delim);
+		while ($st->hasMoreTokens()) {
+			print($st->nextToken(), "\n");
+		}
+		unset($st);
+	}
+}
diff --git a/tests/tokenizer.exp b/tests/tokenizer.exp
new file mode 100644
index 0000000..413f8b1
--- /dev/null
+++ b/tests/tokenizer.exp
@@ -0,0 +1,4 @@
+This
+is
+a
+TEST