word_count   xtlang

(bind-func word_count
  "turn a c string into a word count dict of <String*,i64>, splitting on split_on"
  (lambda (str split_on)
    (let ((split_on_cstr (cstring split_on))
          (tokenptr (strtok (cstring str) split_on_cstr))
          (dict (list (Pair (String "first_key") 0)))
          (cnt 0))
      (while (not (null? tokenptr))
        (if (= (% cnt 10000) 0)
            (printf "cnt: %lld tokenptr: %p\n" cnt tokenptr))
        (set! cnt (+ cnt 1))
        (dict_update dict (String tokenptr) (lambda (x) (+ x 1)))
        (set! tokenptr (strtok null split_on_cstr)))
      (println "processed" cnt "tokens")
      dict)))