Replacement function for list of stuff. about textanalysis.jl

[timClicks]

Perhaps there could be an argument to the TokenDocument constructor that allows for further pre-processing than what is provided by default.

[timClicks]

The following code might be able to be included in preprocessing.jl:

function replace_many(s::String, replacements::Dict{Regex, String})
  for replacement in replacements
        pattern, replacement = replacement
        s = replace(s, pattern, replacement)
  end
  s
end

function replace_many!(d::FileDocument)
    error("FileDocument cannot be modified")
end

function replace_many!(d::StringDocument, replacements::Dict{Regex, String})
    d.text = replace_many(d.text, replacements)
    nothing
end

function replace_many!(d::TokenDocument, replacements::Dict{Regex, String})
    for i in 1:length(d.tokens)
        d.tokens[i] = replace_many(d.tokens[i], replacements)
  end
end

function replace_many!(d::NGramDocument, replacements::Dict{Regex, String})
  for token in keys(d.ngrams)
    new_token = replace_many(new_token, regex, replacements)
    if new_token != token
      if haskey(d.ngrams, new_token)
        d.ngrams[new_token] = pop!(d.ngrams, token) + d.ngrams[new_token]
      else
        d.ngrams[new_token] = pop!(d.ngrams, token)
      end
    end
  end
end

function replace_many!(crps::Corpus, replacements::Dict{Regex, String})
  for doc in crps
    replace_many!(doc)
  end
end

It handles simple replacements quite well, but I'm not sure if Julia's replace function knows how to deal with regular expressions that refer back to groups.

REPLACEMENT_PATTERNS = Dict{Regex, String}(
  r"(?i)won\'t" => "will not",
  r"((?i)can\'t|(?i)can not)" => "cannot",
  r"(?i)i\'m" => "i am",
  r"(?i)ain\'t" => "am not",
  r"\'cause" => "because"

  # doesn't support these
  #r"(\w+)\'ll" => "\g will",
  #r"(\w+)n\'t" => "\g not",
  #r"(\w+)\'ve" => "\g have",
  #r"(\w+t)\'s" => "\g is",
  #r"(\w+)\'re" => "\g are",
  #r"(\w+)\'d" => "\g would",
)

doc = StringDocument("i won't have seen any of this, yet. i'm ready though. i'll be waiting.")
replace_many!(doc, REPLACEMENT_PATTERNS)
text(doc) # =>  "i will not have seen any of this, yet. i am ready though. i will be waiting."