diff --git a/compute-accuracy.c b/compute-accuracy.c new file mode 100644 index 0000000..27bc4a4 --- /dev/null +++ b/compute-accuracy.c @@ -0,0 +1,137 @@ +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include + +const long long max_size = 2000; // max length of strings +const long long N = 1; // number of closest words +const long long max_w = 50; // max length of vocabulary entries + +int main(int argc, char **argv) +{ + FILE *f; + char st1[max_size], st2[max_size], st3[max_size], st4[max_size], bestw[N][max_size], file_name[max_size], ch; + float dist, len, bestd[N], vec[max_size]; + long long words, size, a, b, c, d, b1, b2, b3, threshold = 0; + float *M; + char *vocab; + int TCN, CCN = 0, TACN = 0, CACN = 0, SECN = 0, SYCN = 0, SEAC = 0, SYAC = 0, QID = 0, TQ = 0, TQS = 0; + if (argc < 2) { + printf("Usage: ./compute-accuracy \nwhere FILE contains word projections, and threshold is used to reduce vocabulary of the model for fast approximate evaluation (0 = off, otherwise typical value is 30000)\n"); + return 0; + } + strcpy(file_name, argv[1]); + if (argc > 2) threshold = atoi(argv[2]); + f = fopen(file_name, "rb"); + if (f == NULL) { + printf("Input file not found\n"); + return -1; + } + fscanf(f, "%lld", &words); + if (threshold) if (words > threshold) words = threshold; + fscanf(f, "%lld", &size); + vocab = (char *)malloc(words * max_w * sizeof(char)); + M = (float *)malloc(words * size * sizeof(float)); + if (M == NULL) { + printf("Cannot allocate memory: %lld MB\n", words * size * sizeof(float) / 1048576); + return -1; + } + for (b = 0; b < words; b++) { + fscanf(f, "%s%c", &vocab[b * max_w], &ch); + for (a = 0; a < max_w; a++) vocab[b * max_w + a] = toupper(vocab[b * max_w + a]); + for (a = 0; a < size; a++) fread(&M[a + b * size], sizeof(float), 1, f); + len = 0; + for (a = 0; a < size; a++) len += M[a + b * size] * M[a + b * size]; + len = sqrt(len); + for (a = 0; a < size; a++) M[a + b * size] /= len; + } + fclose(f); + TCN = 0; + while (1) { + for (a = 0; a < N; a++) bestd[a] = 0; + for (a = 0; a < N; a++) bestw[a][0] = 0; + scanf("%s", st1); + for (a = 0; a < strlen(st1); a++) st1[a] = toupper(st1[a]); + if ((!strcmp(st1, ":")) || (!strcmp(st1, "EXIT")) || feof(stdin)) { + if (TCN == 0) TCN = 1; + if (QID != 0) { + printf("ACCURACY TOP1: %.2f %% (%d / %d)\n", CCN / (float)TCN * 100, CCN, TCN); + printf("Total accuracy: %.2f %% Semantic accuracy: %.2f %% Syntactic accuracy: %.2f %% \n", CACN / (float)TACN * 100, SEAC / (float)SECN * 100, SYAC / (float)SYCN * 100); + } + QID++; + scanf("%s", st1); + if (feof(stdin)) break; + printf("%s:\n", st1); + TCN = 0; + CCN = 0; + continue; + } + if (!strcmp(st1, "EXIT")) break; + scanf("%s", st2); + for (a = 0; a < strlen(st2); a++) st2[a] = toupper(st2[a]); + scanf("%s", st3); + for (a = 0; a bestd[a]) { + for (d = N - 1; d > a; d--) { + bestd[d] = bestd[d - 1]; + strcpy(bestw[d], bestw[d - 1]); + } + bestd[a] = dist; + strcpy(bestw[a], &vocab[c * max_w]); + break; + } + } + } + if (!strcmp(st4, bestw[0])) { + CCN++; + CACN++; + if (QID <= 5) SEAC++; else SYAC++; + } + if (QID <= 5) SECN++; else SYCN++; + TCN++; + TACN++; + } + printf("Questions seen / total: %d %d %.2f %% \n", TQS, TQ, TQS/(float)TQ*100); + return 0; +}