Misspellings of Brooklyn
This is an example notebook that uses the SOUNDEX function to identify potential misspellings of the city name BROOKLYN in the NYC DOB Job Application Filings dataset.
[1]:
# Use the 'DOB Job Application Filings - Download' notebook to download the
# 'DOB Job Application Filings' dataset for this example.
datafile = './ic3t-wcy2.tsv.gz'
# As an alternative, you can also use the smaller dataset sample that is
# included in the repository.
#
# datafile = './data/ic3t-wcy2.tsv.gz'
[2]:
# This example makes use of the streaming option that avoids loading the full
# data frame into memory.
from openclean.pipeline import stream
df = stream(datafile)
[3]:
# Find entries where the Soundex of the City is the same as the soundex for 'BROOKLYN'
# but where the city name is not 'BROOKLYN', i.e., potential misspellings.
from openclean.function.eval.base import Col, Eval
from openclean.function.eval.logic import And
from openclean.function.value.phonetic import Soundex, soundex
brooklyn = df\
.select('City ')\
.update('City ', str.upper)\
.filter(And(Eval('City ', Soundex()) == soundex('BROOKLYN'), Col('City ') != 'BROOKLYN'))\
.distinct()
[4]:
# Print (potential) misspellings in decreasing order of their
# frequency.
print('RANK\tCOUNT\tNAME')
for i, entry in enumerate(brooklyn.most_common()):
key, count = entry
print('{}.\t{}\t{}'.format(i + 1, count, key))
RANK COUNT NAME
1. 1059 BRKLYN
2. 749 BROOKYLN
3. 437 BROOKLY
4. 288 BROOKLYN,
5. 231 BROKLYN
6. 162 BRROKLYN
7. 162 BROOKLN
8. 107 BROOKLYLN
9. 84 BROOOKLYN
10. 70 BROOKLNY
11. 56 BROOKLYM
12. 54 BRO0KLYN
13. 49 BROOKKLYN
14. 45 BROOKLYNB
15. 45 BROOKLLYN
16. 35 BROOKLYN NY
17. 35 BROKKLYN
18. 33 BRROOKLYN
19. 29 BROOKLKYN
20. 28 BROOKLYNN
21. 27 BROOKLTN
22. 24 BROOKLYN`
23. 22 BROOKLUN
24. 22 BROOKLYNQ
25. 21 BROOKLINE
26. 20 BROOKLNYN
27. 19 BROOKLYN, NY
28. 13 BROOKLYB
29. 13 BROOKL
30. 12 BROOKLY N
31. 11 BROOKLYKN
32. 11 BERKELEY
33. 11 BROOKJLYN
34. 10 BROOKLYNS
35. 10 BR00KLYN
36. 10 BROOKYLYN
37. 10 BERKLEY
38. 8 BROOKL;YN
39. 8 BROOKLYNM
40. 7 BROOKLYN.
41. 7 BROOKLYTN
42. 7 BROOKLYHN
43. 7 BROOKLIN
44. 7 BROOKLVILLE
45. 6 BROOJLYN
46. 6 BROOKLYN HEIGHT
47. 6 BROOKLYNNY
48. 6 BROOKLOYN
49. 5 BROOKLRN
50. 5 BRIOOKLYN
51. 5 BRIIKLYN
52. 5 BROOKLEN
53. 4 BROOKLYN1
54. 4 BROOKLYN `
55. 4 BORRKLYN
56. 4 BROOKILYN
57. 4 BBROOKLYN
58. 3 BROOKLYBN
59. 3 BROKLY
60. 3 BROOKLYN=
61. 3 BR0OKLYN
62. 3 BROOKL6YN
63. 3 BVROOKLYN
64. 3 BROOKLYN,NY
65. 3 BROOKLYN-
66. 3 BROOKOLYN
67. 3 BROOOKKLYN
68. 3 B ROOKLYN
69. 3 BROOKLYN, NYC
70. 3 BROOKLY6N
71. 3 BROOKLYN, N.Y.
72. 2 BROOKLU
73. 2 BROOKLYNE
74. 2 BROOKLYN 11207
75. 2 BROOKLYNENS
76. 2 BROOKLYIN
77. 2 BROOIKLYN
78. 2 BROOKLTYN
79. 2 BROOKLYH
80. 2 BROOKL YN
81. 2 BROOKLYN11201
82. 2 BROOYKLN
83. 2 BROOK.LYN
84. 2 BROOKLYN01
85. 2 BREOOKLYN
86. 2 BROOKLYNT
87. 2 BRO9KLYN
88. 2 BROOKLKN
89. 2 BROOKLYNH
90. 2 BROOKLYN0
91. 2 BERKLEY HTS.
92. 2 BERKELEY HEIGHT
93. 2 BEROOKLYN
94. 2 BRIACLIFF MANOR
95. 1 BROOKLYN NYC
96. 1 BROOKL.YN
97. 1 BROOKLYN HGTS
98. 1 BROOKLYN11234
99. 1 BROOKLYN``
100. 1 BROOKLYN``````
101. 1 BRKLLYN
102. 1 BFROOKLYN
103. 1 BROOKLAY
104. 1 BROOKLYN{
105. 1 BROO9KLYN
106. 1 BROOKLYN AVE
107. 1 BROOKLYJN
108. 1 BROOJKLYN
109. 1 BROOKLKY
110. 1 BROOKLIYN
111. 1 BRIOKLYN
112. 1 BROOKLYNMASPETH
113. 1 BROOKLYNL
114. 1 BROOKLYNTCHEN
115. 1 BROOKLUM
116. 1 BROOKLYNNAVYYD
117. 1 BROOKYL
118. 1 BROOKLYU
119. 1 BBBROOKLYN
120. 1 BROOAKLYN
121. 1 BROOKLING
122. 1 BROOKLYN NAVY Y
123. 1 BROOKLYN Q
124. 1 B2ROOKLYN
125. 1 BROOKLYON
126. 1 BROOKLYNBROOKLY
127. 1 BROKLYLN
128. 1 BROOKLYN/OZONE
129. 1 BROOKLYNO
130. 1 BROOKLYNB\
131. 1 BROOKLYEN
132. 1 BROOKLYN11238
133. 1 BRROKLY
134. 1 BROOKLYN HTS
135. 1 BROOKLEY
136. 1 BRKLN
137. 1 BROOKLYNHEIGHTS
138. 1 BROOKLYN N.Y. 1
139. 1 BROOKLYN162
140. 1 BRAOOKLYN
141. 1 BROOKLYN+
142. 1 B4ROOKLYN
143. 1 BROOKLNM
144. 1 BROKLY N
145. 1 BROOKLYN P
146. 1 BROOKLYNJ
147. 1 BORSALOM
148. 1 BRKOOLYN
149. 1 BROOKLYNC
150. 1 BROOOKLN
151. 1 BRUUKLYN
152. 1 BRKKLYN
153. 1 BERKELEY HTS.
154. 1 BROOKLYNY
155. 1 BRKLY
156. 1 BERKELEY HTS
157. 1 BROOKLYN11219
158. 1 BRRRKLYN
159. 1 BROOKLYN NY 112
160. 1 BROOK\LYN
161. 1 BROOKLYYN
162. 1 BOORKLY
163. 1 BROOKLYK
164. 1 BERKLEY HEIGHTS
165. 1 BROCKLYN
166. 1 BR4OOKLYN
167. 1 BROOKLYN `
168. 1 BROOKLYMN
169. 1 BROOKLYNEW YORK
170. 1 BROOK;LYN
171. 1 BROOKLN NY