1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
|
# when raw index has a lot of entries like
# 1578324 problematico, a, ci, che
# apply this algorithm:
# treat things after comma as suffixes
# for each suffix:
# if single letter, replace last letter
# else search backwards for beginning of suffix
# and if it leads to an old suffix of approximately
# the same length, put replace that suffix
# This will still leave some commas to fix by hand
# Usage: awk -F' ' -f comfix.awk rawindex > newrawindex
NF == 2 {
i = index($2, ",")
if(i == 0 || length($2) == 0)
print $0
else {
n = split($2, a, /,[ ]*/)
w = a[1]
printf "%s\t%s\n", $1, w
for(i = 2; i <= n; i++) {
suf = a[i]
m = matchsuflen(w, suf)
if(m) {
nw = substr(w, 1, length(w)-m) suf
printf "%s\t%s\n", $1, nw
} else
printf "%s\t%s\n", $1, w ", " suf
}
}
}
NF != 2 {
print $0
}
function matchsuflen(w, suf, wlen,suflen,c,pat,k,d)
{
wlen = length(w)
suflen = length(suf)
if(suflen == 1)
return 1
else {
c = substr(suf, 1, 1)
for (k = 1; k <= wlen ; k++)
if(substr(w, wlen-k+1, 1) == c)
break
if(k > wlen)
return 0
d = k-suflen
if(d < 0)
d = -d
if(d > 3)
return 0
return k
}
}
|