zhdrfirst / word2vec Goto Github PK

If, after making needed corrections, this could be added to the source code, I 
think future users would appreciate this. Thanks. --Gregg Williams

--- begin distance.c ---
//  Copyright 2013 Google Inc. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

#include <stdio.h>
#include <string.h>
#include <math.h>
#include <stdlib.h>

const long long max_size = 2000;         // max length of strings
const long long N = 40;                  // number of closest words that will 
be shown
const long long max_w = 50;              // max length of vocabulary entries

int main(int argc, char **argv) {
  FILE *f;
  char st1[max_size];
  char bestw[N][max_size];
  char file_name[max_size], st[100][max_size];
  float dist, len, bestd[N], vec[max_size];
  long long words, size, a, b, c, d, cn, bi[100];

  char ch;
  float *M;
  char *vocab;
  if (argc < 2) {
    printf("Usage: ./distance <FILE>\nwhere FILE contains word projections in the BINARY FORMAT\n");
    return 0;
  }
  strcpy(file_name, argv[1]);
  f = fopen(file_name, "rb");
  if (f == NULL) {
    printf("Input file not found\n");
    return -1;
  }
  // words = number of words in file
  fscanf(f, "%lld", &words);

  // size = number of floating-point values associated with each word in the "dictionary"
  fscanf(f, "%lld", &size);

  // vocab points to a list of all the words in the "dictionary". Words are stored in fixed-width substrings;
  //    each word is allotted max_w bytes.
  vocab = (char *)malloc((long long)words * max_w * sizeof(char));

  // SUMMARY: M contains 'size' (an integer) floats for each word in the "dictionary".
  // M points to a vector of (words * size) floats, stored linearly. Floats 0 through (size - 1) correspond
  //     to word 0, floats size through (2 * size - 1) correspond to word 1, etc.
  M = (float *)malloc((long long)words * (long long)size * sizeof(float));
  if (M == NULL) {
    printf("Cannot allocate memory: %lld MB    %lld  %lld\n", (long long)words * size * sizeof(float) / 1048576, words, size);
    return -1;
  }

  for (b = 0; b < words; b++) {

    // Reads one entry from input file f, which corresponds to one word of the "dictionary" of words contained in f;
    // this word is stored in the specific substring of vocab reserved for it.
    // UNSURE WHAT PURPOSE SERVED by the char pointed to by &ch--maybe a NUL char denoting end-of-string?
    fscanf(f, "%s%c", &vocab[b * max_w], &ch);

    // Reads 'size' floats, corresponding to the word with index b, into array M.
    for (a = 0; a < size; a++) fread(&M[a + b * size], sizeof(float), 1, f);

    // len = sqrt (sum of [each entry in M] ** 2 ) -- a normalizing factor
    len = 0;
    for (a = 0; a < size; a++) len += M[a + b * size] * M[a + b * size];
    len = sqrt(len);

    // Each entry in M is normalized by a factor of 'len'.
    for (a = 0; a < size; a++) M[a + b * size] /= len;
  }
  fclose(f);

  // **********************************************
  // ****  beginning of user-interaction loop  ****
  // **********************************************
  while (1) {
    for (a = 0; a < N; a++) bestd[a] = 0;
    for (a = 0; a < N; a++) bestw[a][0] = 0;
    printf("Enter word or sentence (EXIT to break): ");

    // st1 contains a input text from stdin (usually the console)
    a = 0;
    while (1) {
      st1[a] = fgetc(stdin);
      if ((st1[a] == '\n') || (a >= max_size - 1)) {
        st1[a] = 0;
        break;
      }
      a++;
    }

    // End program loop if input text = "EXIT".
    if (!strcmp(st1, "EXIT")) break;

    // st[0] is a zero-terminated character array representing the input text.
    cn = 0;
    b = 0;
    c = 0;
    while (1) {
      st[cn][b] = st1[c];
      b++;
      c++;
      st[cn][b] = 0;
      if (st1[c] == 0) break;
      if (st1[c] == ' ') {
        cn++;
        b = 0;
        c++;
      }
    }
    cn++;
    // cn = number of words (separated by a space) in the input text
    // st = an array of strings: st[0][] is the first word of the input text; st[1][] is the second word, etc.

    // This loop either finds each word within the input text in the 'vocab' string, or it signals 
    //     b = -1 if at least one word is not found. If a word is found, b is the index to it in 'vocab'.
    // bi[0] = the index of the first word in the input text; bi[1] = the index of the second word, etc.;
    //     bi[k] = -1 signals no more words--i.e., there are k words in the input text.
    // For each word in the input text, the word and its position in the "dictionary" is printed.
    for (a = 0; a < cn; a++) {
      // 
      for (b = 0; b < words; b++) if (!strcmp(&vocab[b * max_w], st[a])) break;
      if (b == words) b = -1;
      bi[a] = b;
      printf("\nWord: %s  Position in vocabulary: %lld\n", st[a], bi[a]);
      if (b == -1) {
        printf("Out of dictionary word!\n");
        break;
      }
    }

    // Ff input text is not found, restart user-interaction loop.
    if (b == -1) continue;

    // Reminder:
    //     st points to the words in the input text (there are cn of them)
    //     bi[k] is the index of word st[k] within the 'vocab' string
    //     M gives the precalculated "cosine distance" value for the word at st[k][].

    // The code below finds and prints the N "closest" words to the input and their 
    //     "similarity" values (which are always < 1.0)--larger values are "closer".
    printf("\n                                              Word       Cosine distance\n------------------------------------------------------------------------\n");
    // vec contains the 'size' floating-point values associated with the input text.
    // NOTE: if the input text contains multiple words, the value of each element in vec is 
    //       the SUM of the corresponding float values for each of the words in the input text.
    for (a = 0; a < size; a++) vec[a] = 0;
    for (b = 0; b < cn; b++) {
      if (bi[b] == -1) continue;
      // vec contains the 'size' vectors associated with the bi[b]-th word in the "dictionary".
      for (a = 0; a < size; a++) vec[a] += M[a + bi[b] * size];
    }

    // len = sqrt (sum of the squares of each vector element within vec)
    // Each element in vec is normalized by dividing it by 'len'.
    len = 0;
    for (a = 0; a < size; a++) len += vec[a] * vec[a];
    len = sqrt(len);
    for (a = 0; a < size; a++) vec[a] /= len;

    // Arrays bestd and bestw are associated with the list of the N words that are "closest"
    //     to the word(s) in the input text
    // For an index i, bestw[i][] points to the word in that slot,
    //                 bestd[i]   points to the word's "distance" value.
    for (a = 0; a < N; a++) bestd[a] = 0;
    for (a = 0; a < N; a++) bestw[a][0] = 0;

    // For each word in "dictionary"....    (in loop, c is the index of the word being tested)
    for (c = 0; c < words; c++) {
        // a is set to 1 if any of the words in the input text is the word being tested.
      a = 0;
      for (b = 0; b < cn; b++) if (bi[b] == c) a = 1;
      if (a == 1) continue;
      // The following executes only if the word being tested is NOT in the input text.
      dist = 0;
      // dist = sum of the 'size' the float values associated with the word being tested
      for (a = 0; a < size; a++) dist += vec[a] * M[a + c * size];
      // for each of the N slots that will eventually hold the N "closest" words...
      for (a = 0; a < N; a++) {
          // if the "distance" of word c is greater than the "distance" of the current slot (slot a),
          //     move all the bestd and bestw entries one entry closer to the end of the list (losing
          //     the "worst" entry) and insert the bestd and bestw entries for the current word (c)
          //     into the current slot (a).
        if (dist > bestd[a]) {
          for (d = N - 1; d > a; d--) {
            bestd[d] = bestd[d - 1];
            strcpy(bestw[d], bestw[d - 1]);
          }
          bestd[a] = dist;
          strcpy(bestw[a], &vocab[c * max_w]);
          break;
        }
      }
    }
    // From "best" to "worst", print each word and its "distance" value.
    for (a = 0; a < N; a++) printf("%50s\t\t%f\n", bestw[a], bestd[a]);
  }
  return 0;
}
---  end distance.c  ---

word2vec does not free allocated objects correctly. It also reads freed objects.
Attached patch fixes this issue.

What steps will reproduce the problem?
On a Mac:
1. svn checkout http://word2vec.googlecode.com/svn/trunk/
2. make

What is the expected output?
Binary is emitted.

What do you see instead?
pindari:word2vec pmonks$ make
gcc word2vec.c -o word2vec -lm -pthread -Ofast -march=native -Wall 
-funroll-loops -Wno-unused-result
cc1: error: invalid option argument ‘-Ofast’
cc1: error: unrecognized command line option "-Wno-unused-result"
word2vec.c:1: error: bad value (native) for -march= switch
word2vec.c:1: error: bad value (native) for -mtune= switch
make: *** [word2vec] Error 1
pindari:word2vec pmonks$

What version of the product are you using?
SVN r32

On what operating system?
Mac OSX 10.8.4

$ ./demo-phrase-accuracy.sh 
make: Nothing to be done for `all'.
Starting training using file text8
Words processed: 17000K     Vocab size: 4399K  
Vocab size (unigrams + bigrams): 2586139
Words in train file: 17005206
Words written: 17000K
real    0m21.130s
user    0m20.062s
sys 0m1.054s
Starting training using file text8-phrase
Vocab size: 123636
Words in train file: 16337523
Alpha: 0.000119  Progress: 99.59%  Words/thread/sec: 22.70k  
real    1m38.617s
user    12m0.795s
sys 0m1.501s
newspapers:
./demo-phrase-accuracy.sh: line 12: 36538 Segmentation fault: 11  
./compute-accuracy vectors-phrase.bin < questions-phrases.txt

I'm on OSX (latest non-beta), and had to switch out "#include <stdlib.h>" to 
get it to compile, but no other changes.

Sorry to put a plain old question here, but... is there a correct place to put 
a plain old question?
In other words, is there a way to engaging in dialog about this project? I have 
several points I'd like to discuss.