From 3ea1dff1c384914dcfc78e565bd962abfd2525fd Mon Sep 17 00:00:00 2001 From: Christiaan van Luik Date: Fri, 7 Aug 2026 14:52:03 +0200 Subject: [PATCH 1/5] Add get_admin1_codes() for first-level administrative divisions Adds the GeoNames admin1CodesASCII.txt dataset as a new admin1.json data file, keyed by the composite code . (e. g. US.CA). Since city records store countrycode and admin1code separately, the composite key allows resolving those references to the admin1 name and geonameid. --- CHANGELOG.md | 6 ++++++ Makefile | 1 + README.md | 5 ++++- bin/admin1.py | 24 ++++++++++++++++++++++++ bin/download_data.py | 1 + geonamescache/__init__.py | 7 +++++++ geonamescache/types.py | 7 +++++++ tests/test_geonamescache.py | 20 ++++++++++++++++++++ 8 files changed, 70 insertions(+), 1 deletion(-) create mode 100755 bin/admin1.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 68cc655..28aa43a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- Add `get_admin1_codes()` method returning first-level administrative division data from the GeoNames `admin1CodesASCII.txt` dataset, keyed by `.` (e. g. `US.CA`), which allows resolving the `countrycode`/`admin1code` references stored in city records. + ## [3.0.2](https://github.com/yaph/geonamescache/releases/tag/3.0.2) - 2026-07-28 [Compare with 3.0.1](https://github.com/yaph/geonamescache/compare/3.0.1...3.0.2) diff --git a/Makefile b/Makefile index 3ff4b16..f8715fb 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ dl: json: mkdir -p geonamescache/data/ + ./bin/admin1.py ./bin/continents.py ./bin/countries.py ./bin/cities.py diff --git a/README.md b/README.md index 9c37924..eee28e0 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![image](https://img.shields.io/pypi/v/geonamescache.svg)](https://pypi.python.org/pypi/geonamescache) -A Python library that provides functions to retrieve names, ISO and FIPS codes of continents, countries as well as US states and counties as Python dictionaries. The country and city datasets also include population and geographic data. +A Python library that provides functions to retrieve names, ISO and FIPS codes of continents, countries and first-level administrative divisions as well as US states and counties as Python dictionaries. The country and city datasets also include population and geographic data. Geonames data is obtained from [GeoNames](http://www.geonames.org/). @@ -31,6 +31,7 @@ Currently geonamescache provides the following methods, that return dictionaries * get\_continents() * get\_countries() +* get\_admin1\_codes() * get\_us\_states() * get\_cities() * get\_countries\_by\_names() @@ -38,6 +39,8 @@ Currently geonamescache provides the following methods, that return dictionaries * get\_cities\_by\_name(name) * get\_us\_counties() +The dictionary returned by `get_admin1_codes()` is keyed by the code `.`, for example `US.CA` for California, which allows resolving the `countrycode` and `admin1code` references stored in city records. + In addition you can search for cities by name. * search\_cities(\'NAME\', case\_sensitive=True, contains\_search=True) diff --git a/bin/admin1.py b/bin/admin1.py new file mode 100755 index 0000000..a66f0fb --- /dev/null +++ b/bin/admin1.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +import csv +import json +from pathlib import Path + +p_data = Path('datasets') + +admin1 = {} + +reader = csv.reader(p_data.joinpath('admin1CodesASCII.txt').open(encoding='utf-8'), 'excel-tab') +for record in reader: + code, name, asciiname, geonameid = record + + # required because used as key + if not code: + continue + + admin1[code] = { + 'asciiname': asciiname, + 'geonameid': int(geonameid) if geonameid else 0, + 'name': name, + } + +p_data.joinpath('admin1.json').write_text(json.dumps(admin1, ensure_ascii=False)) diff --git a/bin/download_data.py b/bin/download_data.py index 57c5932..2817992 100755 --- a/bin/download_data.py +++ b/bin/download_data.py @@ -8,6 +8,7 @@ # Data files to download DOWNLOADS = [ + 'http://download.geonames.org/export/dump/admin1CodesASCII.txt', 'http://download.geonames.org/export/dump/cities500.zip', 'http://download.geonames.org/export/dump/cities1000.zip', 'http://download.geonames.org/export/dump/cities5000.zip', diff --git a/geonamescache/__init__.py b/geonamescache/__init__.py index 20f2a43..5ca2f83 100644 --- a/geonamescache/__init__.py +++ b/geonamescache/__init__.py @@ -10,6 +10,8 @@ from typing import Any, ClassVar, TypeVar from geonamescache.types import ( + Admin1, + Admin1CodeStr, City, CitySearchAttribute, Continent, @@ -27,6 +29,7 @@ class GeonamesCache: + admin1: dict[Admin1CodeStr, Admin1] | None = None continents: dict[ContinentCode, Continent] | None = None countries: dict[ISOStr, Country] | None = None cities: dict[GeoNameIdStr, City] | None = None @@ -47,6 +50,10 @@ def get_continents(self) -> dict[ContinentCode, Continent]: def get_countries(self) -> dict[ISOStr, Country]: return self._load_data(self.countries, 'countries.json') + def get_admin1_codes(self) -> dict[Admin1CodeStr, Admin1]: + """Get first-level administrative divisions keyed by ., e. g. US.CA.""" + return self._load_data(self.admin1, 'admin1.json') + def get_us_states(self) -> dict[USStateCode, USState]: return self._load_data(self.us_states, 'us_states.json') diff --git a/geonamescache/types.py b/geonamescache/types.py index 5a700d1..ecf282a 100644 --- a/geonamescache/types.py +++ b/geonamescache/types.py @@ -8,6 +8,7 @@ GeoNameIdStr = str ISOStr = str +Admin1CodeStr = str ContinentCode = Literal["AF", "AN", "AS", "EU", "NA", "OC", "SA"] USStateCode = Literal[ "AK", @@ -167,6 +168,12 @@ class Continent(TypedDict): cc2: NotRequired[str] +class Admin1(TypedDict): + asciiname: str + geonameid: int + name: str + + class City(TypedDict): alternatenames: list[str] admin1code: str diff --git a/tests/test_geonamescache.py b/tests/test_geonamescache.py index 123dbc1..fe68633 100644 --- a/tests/test_geonamescache.py +++ b/tests/test_geonamescache.py @@ -3,6 +3,26 @@ gc = GeonamesCache() +def test_get_admin1_codes(): + admin1 = gc.get_admin1_codes() + assert len(admin1) > 3000 + for key, name, geonameid in ( + ('US.CA', 'California', 5332921), + ('ES.51', 'Andalusia', 2593109), + ): + assert name == admin1[key]['name'] + assert geonameid == admin1[key]['geonameid'] + + +def test_admin1_code_resolves_city_reference(): + # Cities store countrycode and admin1code separately, the composite + # admin1 key allows resolving these references. + city = gc.get_cities()['5368361'] + assert 'Los Angeles' == city['name'] + key = f"{city['countrycode']}.{city['admin1code']}" + assert 'California' == gc.get_admin1_codes()[key]['name'] + + def test_get_countries_by_names(): # Length of get_countries_by_names dict and get_countries dict must be # the same, unless country names wouldn't be unique. From 06f51173e82a3edb40f917447734723d72d22f77 Mon Sep 17 00:00:00 2001 From: Christiaan van Luik Date: Wed, 19 Aug 2026 10:12:47 +0200 Subject: [PATCH 2/5] Add admin2 and timezone datasets, fix dataset caching Adds two GeoNames datasets and the helpers needed to join them to city records, plus fixes to the caching that made repeated lookups re-read their JSON files and could serve results from the wrong dataset. Datasets: - admin2Codes.txt becomes admin2.json, keyed by the concatenated code .. as described in the GeoNames readme, exposed through get_admin2_codes(). - timeZones.txt becomes timezones.json, keyed by IANA time zone id, exposed through get_timezones() and get_timezones_by_country(). City records gain admin2code, completing the composite key. The resolved admin1 name is deliberately not stored on city records: denormalising it grew cities500.json by 12 MB, so get_admin1_by_city() and get_admin2_by_city() do the join instead. Both return None rather than building a partial key such as 'NL.' when a city has no admin1code. Caching fixes: - _load_data() returned the parsed data without storing it, so every getter re-read and re-parsed its file. Repeated get_cities() calls on the 500 population dataset cost ~0.45 s each and are now free after the first. _load_data() is now only the file reader and its return type no longer claims dict, which was wrong for us_counties.json. - cities_by_names was a class attribute keyed by city name alone, while the results depend on min_city_population, so an instance created after one with a smaller dataset was served the other's results. It is now per instance. The README gains a data formats section documenting the return value of every method. Every example in it was run against the built data. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 11 +++ Makefile | 2 + README.md | 180 +++++++++++++++++++++++++++++++++++- bin/admin2.py | 24 +++++ bin/cities.py | 1 + bin/download_data.py | 2 + bin/timezones.py | 31 +++++++ geonamescache/__init__.py | 91 +++++++++++++++--- geonamescache/types.py | 19 +++- tests/test_geonamescache.py | 116 +++++++++++++++++++++++ 10 files changed, 459 insertions(+), 18 deletions(-) create mode 100755 bin/admin2.py create mode 100755 bin/timezones.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 28aa43a..b102c66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,20 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### Fixed + +- Fix dataset caching. `_load_data()` returned the parsed data without storing it, so every getter call re-read and re-parsed its JSON file from disk. Repeated `get_cities()` calls on the 500-population dataset took roughly 0.4 s each and now cost nothing after the first. +- Fix `get_cities_by_name()` returning results from the wrong dataset. Its cache was a class attribute keyed by city name only, while results depend on `min_city_population`, so an instance created after one with a smaller dataset was served the other instance's results. The cache is now per instance. + ### Added - Add `get_admin1_codes()` method returning first-level administrative division data from the GeoNames `admin1CodesASCII.txt` dataset, keyed by `.` (e. g. `US.CA`), which allows resolving the `countrycode`/`admin1code` references stored in city records. +- Add `get_admin2_codes()` method returning second-level administrative division data from the GeoNames `admin2Codes.txt` dataset, keyed by `..` (e. g. `NL.11.0599`). +- Add `admin2code` to city records, completing the composite key needed to look up second-level divisions. +- Add `get_admin1_by_city()` and `get_admin2_by_city()` methods that resolve the administrative division of a city record, returning `None` when the city's codes are missing or absent from the division dataset. +- Add `get_timezones()` method returning time zone data from the GeoNames `timeZones.txt` dataset, keyed by IANA time zone id (e. g. `Europe/Amsterdam`), which resolves the `timezone` field stored in city records. +- Add `get_timezones_by_country()` method returning the time zones of a country as a list sorted by time zone id, taking a case insensitive ISO alpha-2 country code. +- Add a data formats section to the README documenting the return value of every method with concrete examples. ## [3.0.2](https://github.com/yaph/geonamescache/releases/tag/3.0.2) - 2026-07-28 diff --git a/Makefile b/Makefile index f8715fb..f7dfc62 100644 --- a/Makefile +++ b/Makefile @@ -14,11 +14,13 @@ dl: json: mkdir -p geonamescache/data/ ./bin/admin1.py + ./bin/admin2.py ./bin/continents.py ./bin/countries.py ./bin/cities.py ./bin/us_counties.py ./bin/us_states.py + ./bin/timezones.py mv datasets/*.json geonamescache/data/ clean: clean-build clean-py clean-dev clean-datasets diff --git a/README.md b/README.md index eee28e0..b73b08a 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![image](https://img.shields.io/pypi/v/geonamescache.svg)](https://pypi.python.org/pypi/geonamescache) -A Python library that provides functions to retrieve names, ISO and FIPS codes of continents, countries and first-level administrative divisions as well as US states and counties as Python dictionaries. The country and city datasets also include population and geographic data. +A Python library that provides functions to retrieve names, ISO and FIPS codes of continents, countries and first- and second-level administrative divisions as well as US states and counties as Python dictionaries. The country and city datasets also include population and geographic data. Geonames data is obtained from [GeoNames](http://www.geonames.org/). @@ -19,6 +19,8 @@ A simple example: gc = geonamescache.GeonamesCache() print(gc.get_countries()) +Each `GeonamesCache` instance caches every dataset it loads, so keep one instance around rather than creating a new one per lookup. + ## Settings ### Cities dataset @@ -32,14 +34,14 @@ Currently geonamescache provides the following methods, that return dictionaries * get\_continents() * get\_countries() * get\_admin1\_codes() +* get\_admin2\_codes() * get\_us\_states() * get\_cities() * get\_countries\_by\_names() * get\_us\_states\_by\_names() * get\_cities\_by\_name(name) * get\_us\_counties() - -The dictionary returned by `get_admin1_codes()` is keyed by the code `.`, for example `US.CA` for California, which allows resolving the `countrycode` and `admin1code` references stored in city records. +* get\_timezones() In addition you can search for cities by name. @@ -51,6 +53,178 @@ This function returns a list of city records that match the given `NAME`. * By default the search is case insensitive, it can be made case sensitive by changing `case_sensitive` to True. * By default the search is contains, it can be made exact equality by changing `contains_search` to False. +To resolve the administrative division a city belongs to, use: + +* get\_admin1\_by\_city(city) +* get\_admin2\_by\_city(city) + +Both take a city record and return the matching division record, or `None` if the city's codes are missing or not present in the division dataset. + +To get the time zones of a country, use: + +* get\_timezones\_by\_country(countrycode) + +## Data formats + +All examples below assume `gc = geonamescache.GeonamesCache()`. + +### get_continents() + +A dictionary keyed by the two-letter continent code. Records come from the GeoNames web service and contain more fields than shown here. + + >>> gc.get_continents()['EU']['name'] + 'Europe' + +### get_countries() + +A dictionary of 252 countries keyed by ISO alpha-2 code. + + >>> gc.get_countries()['US'] + { + 'geonameid': 6252001, + 'name': 'United States', + 'iso': 'US', + 'iso3': 'USA', + 'isonumeric': 840, + 'fips': 'US', + 'continentcode': 'NA', + 'capital': 'Washington', + 'areakm2': 9629091, + 'population': 327167434, + 'tld': '.us', + 'currencycode': 'USD', + 'currencyname': 'Dollar', + 'phone': '1', + 'postalcoderegex': '^\\d{5}(-\\d{4})?$', + 'languages': 'en-US,es-US,haw,fr', + 'neighbours': 'CA,MX,CU' + } + +`get_countries_by_names()` returns the same records keyed by country name instead, e. g. `gc.get_countries_by_names()['Spain']`. + +### get_cities() + +A dictionary keyed by geonameid **as a string**, holding 34078 cities at the default minimum population of 15000. + + >>> gc.get_cities()['2747891'] + { + 'geonameid': 2747891, + 'name': 'Rotterdam', + 'latitude': 51.9225, + 'longitude': 4.47917, + 'countrycode': 'NL', + 'population': 868135, + 'timezone': 'Europe/Amsterdam', + 'admin1code': '11', + 'admin2code': '0599', + 'alternatenames': ['RTM', 'Ratehrdam', 'Roterdam', ...] + } + +City names are not unique, so `get_cities_by_name()` returns a list of single-entry dictionaries rather than one record. There is a Rotterdam in both the Netherlands and the US state of New York: + + >>> [list(d) for d in gc.get_cities_by_name('Rotterdam')] + [['2747891'], ['5134453']] + +`search_cities()` returns a flat list of city records instead. It searches `alternatenames` by default, so it matches places whose *other* names contain the query, here the Rotterdam district of Hoogvliet: + + >>> [(c['name'], c['countrycode']) for c in gc.search_cities('Rotterdam')] + [('Rotterdam', 'NL'), ('Hoogvliet', 'NL')] + +Pass `attribute='name'` to search the primary name instead, which finds the US Rotterdam that has no alternate names: + + >>> [(c['name'], c['countrycode']) for c in gc.search_cities('Rotterdam', attribute='name')] + [('Rotterdam', 'NL'), ('Rotterdam', 'US')] + +### get_admin1_codes() + +First-level administrative divisions (states, provinces, regions), 3865 records keyed by the composite code `.`, for example `US.CA` for California or `NL.11` for South Holland. + + >>> gc.get_admin1_codes()['NL.11'] + {'asciiname': 'South Holland', 'geonameid': 2743698, 'name': 'South Holland'} + +### get_admin2_codes() + +Second-level administrative divisions (counties, municipalities, districts), 47592 records keyed by `..`. + + >>> gc.get_admin2_codes()['NL.11.0599'] + {'asciiname': 'Rotterdam', 'geonameid': 2747890, 'name': 'Rotterdam'} + +Note the `geonameid` here is the municipality of Rotterdam (2747890), which is a different place from the city of Rotterdam (2747891). + +### get_admin1_by_city() and get_admin2_by_city() + +Cities store `countrycode`, `admin1code` and `admin2code` separately, so resolving a division means joining them into the composite key. These helpers do that and handle the cases where a city has no code: + + >>> city = gc.get_cities()['2747891'] + >>> gc.get_admin1_by_city(city)['name'] + 'South Holland' + >>> gc.get_admin2_by_city(city)['name'] + 'Rotterdam' + +Both return `None` when the city lacks the required codes or the composite key is not in the division dataset, which is why the return value should be checked before subscripting it: + + admin1 = gc.get_admin1_by_city(city) + region = admin1['name'] if admin1 else 'unknown' + +Building the key by hand works too, but silently produces a partial key such as `'NL.'` for cities without an admin1code, so prefer the helpers. + +### get_timezones() + +Time zones with their UTC offsets, 418 records keyed by IANA time zone id. + + >>> gc.get_timezones()['Europe/Amsterdam'] + { + 'countrycode': 'NL', + 'timezoneid': 'Europe/Amsterdam', + 'gmtoffset': 1.0, + 'dstoffset': 2.0, + 'rawoffset': 1.0 + } + +`rawoffset` is the offset excluding daylight saving time. `gmtoffset` and `dstoffset` are the offsets in effect on 1 January and 1 July of the year the dataset was published, so they are a snapshot rather than a live value; use a proper time zone library such as `zoneinfo` if you need the offset at a given moment. + +The `timezone` field of every city record is a key into this dictionary: + + >>> city = gc.get_cities()['2747891'] + >>> gc.get_timezones()[city['timezone']]['rawoffset'] + 1.0 + +### get_timezones_by_country(countrycode) + +The time zones of one country as a list sorted by time zone id. The country code is an ISO alpha-2 code and is matched case insensitively. + + >>> [tz['timezoneid'] for tz in gc.get_timezones_by_country('NL')] + ['Europe/Amsterdam'] + + >>> len(gc.get_timezones_by_country('US')) + 29 + +Unknown country codes return an empty list rather than raising: + + >>> gc.get_timezones_by_country('ZZ') + [] + +### get_us_states() + +A dictionary keyed by the two-letter state code. + + >>> gc.get_us_states()['CA'] + {'code': 'CA', 'name': 'California', 'fips': '06', 'geonameid': 5332921} + +`get_us_states_by_names()` returns the same records keyed by state name, e. g. `gc.get_us_states_by_names()['California']`. + +### get_us_counties() + +A **list** of 3235 county records, not a dictionary, sourced from the US Census Bureau rather than GeoNames. + + >>> gc.get_us_counties()[0] + {'fips': '01001', 'name': 'Autauga County', 'state': 'AL'} + +To look counties up, key the list yourself: + + counties = {c['fips']: c for c in gc.get_us_counties()} + counties['06037']['name'] # 'Los Angeles County' + ## Mappers The mappers module provides function(s) to map data properties. Currently you can create a mapper that maps country properties, e. g. the `name` property to the `iso3` property, to do so you'd write the following code: diff --git a/bin/admin2.py b/bin/admin2.py new file mode 100755 index 0000000..a4514ad --- /dev/null +++ b/bin/admin2.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +import csv +import json +from pathlib import Path + +p_data = Path('datasets') + +admin2 = {} + +reader = csv.reader(p_data.joinpath('admin2Codes.txt').open(encoding='utf-8'), 'excel-tab') +for record in reader: + code, name, asciiname, geonameid = record + + # required because used as key + if not code: + continue + + admin2[code] = { + 'asciiname': asciiname, + 'geonameid': int(geonameid) if geonameid else 0, + 'name': name, + } + +p_data.joinpath('admin2.json').write_text(json.dumps(admin2, ensure_ascii=False)) diff --git a/bin/cities.py b/bin/cities.py index f86360b..ccad13e 100755 --- a/bin/cities.py +++ b/bin/cities.py @@ -46,6 +46,7 @@ 'population': int(population), 'timezone': timezone, 'admin1code': admin1code, + 'admin2code': admin2code, 'alternatenames': alternatenames.split(','), } diff --git a/bin/download_data.py b/bin/download_data.py index 2817992..c38d78c 100755 --- a/bin/download_data.py +++ b/bin/download_data.py @@ -9,11 +9,13 @@ # Data files to download DOWNLOADS = [ 'http://download.geonames.org/export/dump/admin1CodesASCII.txt', + 'http://download.geonames.org/export/dump/admin2Codes.txt', 'http://download.geonames.org/export/dump/cities500.zip', 'http://download.geonames.org/export/dump/cities1000.zip', 'http://download.geonames.org/export/dump/cities5000.zip', 'http://download.geonames.org/export/dump/cities15000.zip', 'http://download.geonames.org/export/dump/countryInfo.txt', + 'http://download.geonames.org/export/dump/timeZones.txt', 'https://www2.census.gov/geo/docs/reference/codes2020/national_county2020.txt' ] diff --git a/bin/timezones.py b/bin/timezones.py new file mode 100755 index 0000000..98a5694 --- /dev/null +++ b/bin/timezones.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +import csv +import json +from pathlib import Path + +p_data = Path('datasets') + +timezones = {} + +reader = csv.reader(p_data.joinpath('timeZones.txt').open(encoding='utf-8'), 'excel-tab') +for record in reader: + countrycode, timezoneid, gmtoffset, dstoffset, rawoffset = record + + # The header row names the offset columns after the current year, so match + # on the first column instead, which is stable. + if countrycode == 'CountryCode': + continue + + # required because used as key + if not timezoneid: + continue + + timezones[timezoneid] = { + 'countrycode': countrycode, + 'timezoneid': timezoneid, + 'gmtoffset': float(gmtoffset), + 'dstoffset': float(dstoffset), + 'rawoffset': float(rawoffset), + } + +p_data.joinpath('timezones.json').write_text(json.dumps(timezones, ensure_ascii=False)) diff --git a/geonamescache/__init__.py b/geonamescache/__init__.py index 5ca2f83..ab720ef 100644 --- a/geonamescache/__init__.py +++ b/geonamescache/__init__.py @@ -7,11 +7,13 @@ import json import os from collections.abc import Mapping -from typing import Any, ClassVar, TypeVar +from typing import Any, TypeVar from geonamescache.types import ( Admin1, Admin1CodeStr, + Admin2, + Admin2CodeStr, City, CitySearchAttribute, Continent, @@ -19,6 +21,8 @@ Country, GeoNameIdStr, ISOStr, + TimeZoneIdStr, + TimeZoneInfo, USCounty, USState, USStateCode, @@ -30,32 +34,88 @@ class GeonamesCache: admin1: dict[Admin1CodeStr, Admin1] | None = None + admin2: dict[Admin2CodeStr, Admin2] | None = None continents: dict[ContinentCode, Continent] | None = None countries: dict[ISOStr, Country] | None = None cities: dict[GeoNameIdStr, City] | None = None cities_items: list[tuple[GeoNameIdStr, City]] | None = None - cities_by_names: ClassVar[dict[str, list[dict[GeoNameIdStr, City]]]] = {} + timezones: dict[TimeZoneIdStr, TimeZoneInfo] | None = None us_counties: list[USCounty] | None = None us_states: dict[USStateCode, USState] | None = None def __init__(self, min_city_population: int = 15000): self.min_city_population = min_city_population + # Per instance, because the cache key is the city name alone while the + # results depend on min_city_population. + self.cities_by_names: dict[str, list[dict[GeoNameIdStr, City]]] = {} def get_dataset_by_key(self, dataset: dict[Any, TDict], key: str) -> dict[Any, TDict]: return {d[key]: d for c, d in list(dataset.items())} def get_continents(self) -> dict[ContinentCode, Continent]: - return self._load_data(self.continents, 'continents.json') + if self.continents is None: + self.continents = self._load_data('continents.json') + return self.continents def get_countries(self) -> dict[ISOStr, Country]: - return self._load_data(self.countries, 'countries.json') + if self.countries is None: + self.countries = self._load_data('countries.json') + return self.countries def get_admin1_codes(self) -> dict[Admin1CodeStr, Admin1]: """Get first-level administrative divisions keyed by ., e. g. US.CA.""" - return self._load_data(self.admin1, 'admin1.json') + if self.admin1 is None: + self.admin1 = self._load_data('admin1.json') + return self.admin1 + + def get_admin2_codes(self) -> dict[Admin2CodeStr, Admin2]: + """Get second-level administrative divisions keyed by .., e. g. NL.11.0599.""" + if self.admin2 is None: + self.admin2 = self._load_data('admin2.json') + return self.admin2 + + def get_admin1_by_city(self, city: City) -> Admin1 | None: + """Get the first-level administrative division a city belongs to, None if unresolvable. + + Not every city record has an admin1code, and not every code pair is + present in the admin1 dataset, so callers should handle None. + """ + if not city.get('countrycode') or not city.get('admin1code'): + return None + return self.get_admin1_codes().get(f"{city['countrycode']}.{city['admin1code']}") + + def get_admin2_by_city(self, city: City) -> Admin2 | None: + """Get the second-level administrative division a city belongs to, None if unresolvable. + + Requires all three of countrycode, admin1code and admin2code, as the + admin2 dataset is keyed by the concatenation of them. + """ + if not city.get('countrycode') or not city.get('admin1code') or not city.get('admin2code'): + return None + return self.get_admin2_codes().get(f"{city['countrycode']}.{city['admin1code']}.{city['admin2code']}") + + def get_timezones(self) -> dict[TimeZoneIdStr, TimeZoneInfo]: + """Get time zones keyed by IANA time zone id, e. g. Europe/Amsterdam.""" + if self.timezones is None: + self.timezones = self._load_data('timezones.json') + return self.timezones + + def get_timezones_by_country(self, countrycode: str) -> list[TimeZoneInfo]: + """Get the time zones of a country as a list sorted by time zone id. + + Takes an ISO alpha-2 country code, case insensitive. Returns an empty + list for unknown codes. + """ + countrycode = countrycode.upper() + return sorted( + (tz for tz in self.get_timezones().values() if tz['countrycode'] == countrycode), + key=lambda tz: tz['timezoneid'], + ) def get_us_states(self) -> dict[USStateCode, USState]: - return self._load_data(self.us_states, 'us_states.json') + if self.us_states is None: + self.us_states = self._load_data('us_states.json') + return self.us_states def get_countries_by_names(self) -> dict[str, Country]: return self.get_dataset_by_key(self.get_countries(), 'name') @@ -65,7 +125,9 @@ def get_us_states_by_names(self) -> dict[USStateName, USState]: def get_cities(self) -> dict[GeoNameIdStr, City]: """Get a dictionary of cities keyed by geonameid.""" - return self._load_data(self.cities, f'cities{self.min_city_population}.json') + if self.cities is None: + self.cities = self._load_data(f'cities{self.min_city_population}.json') + return self.cities def get_cities_by_name(self, name: str) -> list[dict[GeoNameIdStr, City]]: """Get a list of city dictionaries with the given name. @@ -79,8 +141,10 @@ def get_cities_by_name(self, name: str) -> list[dict[GeoNameIdStr, City]]: self.cities_by_names[name] = [{gid: city} for gid, city in self.cities_items if city['name'] == name] return self.cities_by_names[name] - def get_us_counties(self): - return self._load_data(self.us_counties, 'us_counties.json') + def get_us_counties(self) -> list[USCounty]: + if self.us_counties is None: + self.us_counties = self._load_data('us_counties.json') + return self.us_counties def search_cities( self, @@ -112,8 +176,7 @@ def search_cities( return results @staticmethod - def _load_data(datadict: dict[Any, Any] | None, datafile: str) -> dict[Any, Any]: - if datadict is None: - with open(os.path.join(os.path.dirname(__file__), 'data', datafile)) as f: - datadict = json.load(f) - return datadict + def _load_data(datafile: str) -> Any: + """Read and parse a bundled data file. Callers are responsible for caching.""" + with open(os.path.join(os.path.dirname(__file__), 'data', datafile), encoding='utf-8') as f: + return json.load(f) diff --git a/geonamescache/types.py b/geonamescache/types.py index ecf282a..f40cf92 100644 --- a/geonamescache/types.py +++ b/geonamescache/types.py @@ -9,6 +9,8 @@ GeoNameIdStr = str ISOStr = str Admin1CodeStr = str +Admin2CodeStr = str +TimeZoneIdStr = str ContinentCode = Literal["AF", "AN", "AS", "EU", "NA", "OC", "SA"] USStateCode = Literal[ "AK", @@ -116,7 +118,7 @@ "West Virginia", "Wyoming", ] -CitySearchAttribute = Literal["alternatenames", "admin1code", "countrycode", "name", "timezone"] +CitySearchAttribute = Literal["alternatenames", "admin1code", "admin2code", "countrycode", "name", "timezone"] class TimeZone(TypedDict): @@ -125,6 +127,14 @@ class TimeZone(TypedDict): timeZoneId: str +class TimeZoneInfo(TypedDict): + countrycode: str + timezoneid: str + gmtoffset: float + dstoffset: float + rawoffset: float + + class BBox(TypedDict): accuracyLevel: int east: float @@ -174,9 +184,16 @@ class Admin1(TypedDict): name: str +class Admin2(TypedDict): + asciiname: str + geonameid: int + name: str + + class City(TypedDict): alternatenames: list[str] admin1code: str + admin2code: str countrycode: str geonameid: int latitude: float diff --git a/tests/test_geonamescache.py b/tests/test_geonamescache.py index fe68633..e122ee2 100644 --- a/tests/test_geonamescache.py +++ b/tests/test_geonamescache.py @@ -23,6 +23,87 @@ def test_admin1_code_resolves_city_reference(): assert 'California' == gc.get_admin1_codes()[key]['name'] +def test_get_admin2_codes(): + admin2 = gc.get_admin2_codes() + assert len(admin2) > 40000 + for key, name, geonameid in ( + ('NL.11.0599', 'Rotterdam', 2747890), + ('US.CA.037', 'Los Angeles County', 5368381), + ): + assert name == admin2[key]['name'] + assert geonameid == admin2[key]['geonameid'] + + +def test_get_admin1_by_city(): + city = gc.get_cities()['2747891'] + assert 'Rotterdam' == city['name'] + assert '11' == city['admin1code'] + admin1 = gc.get_admin1_by_city(city) + assert admin1 is not None + assert 'South Holland' == admin1['name'] + + +def test_get_admin2_by_city(): + city = gc.get_cities()['2747891'] + admin2 = gc.get_admin2_by_city(city) + assert admin2 is not None + assert 'Rotterdam' == admin2['name'] + assert 2747890 == admin2['geonameid'] + + +def test_get_admin_by_city_unresolvable(): + # Missing or unknown codes must return None rather than raise or build a + # partial key such as 'NL.'. + city = dict(gc.get_cities()['2747891']) + for field in ('countrycode', 'admin1code', 'admin2code'): + broken = {**city, field: ''} + assert gc.get_admin2_by_city(broken) is None + assert gc.get_admin1_by_city({**city, 'admin1code': ''}) is None + assert gc.get_admin1_by_city({**city, 'admin1code': 'ZZ'}) is None + + +def test_admin2_code_resolves_city_reference(): + # Cities store countrycode, admin1code and admin2code separately, the + # composite admin2 key allows resolving these references. + city = gc.get_cities()['2747891'] + key = f"{city['countrycode']}.{city['admin1code']}.{city['admin2code']}" + assert 'NL.11.0599' == key + assert 'Rotterdam' == gc.get_admin2_codes()[key]['name'] + + +def test_get_timezones(): + timezones = gc.get_timezones() + assert len(timezones) > 400 + amsterdam = timezones['Europe/Amsterdam'] + assert 'NL' == amsterdam['countrycode'] + assert 1.0 == amsterdam['rawoffset'] + + +def test_get_timezones_by_country(): + assert ['Europe/Amsterdam'] == [tz['timezoneid'] for tz in gc.get_timezones_by_country('NL')] + + # The US spans many zones, the list must be sorted by time zone id. + us = [tz['timezoneid'] for tz in gc.get_timezones_by_country('US')] + assert len(us) > 20 + assert us == sorted(us) + assert 'Pacific/Honolulu' in us + assert all('US' == tz['countrycode'] for tz in gc.get_timezones_by_country('US')) + + +def test_get_timezones_by_country_edge_cases(): + # Country codes are matched case insensitively. + assert gc.get_timezones_by_country('nl') == gc.get_timezones_by_country('NL') + + # Unknown codes return an empty list rather than raising. + assert [] == gc.get_timezones_by_country('ZZ') + + +def test_city_timezone_is_in_timezones(): + # Every timezone referenced by a city record must exist in the dataset. + timezones = gc.get_timezones() + assert all(city['timezone'] in timezones for city in gc.get_cities().values()) + + def test_get_countries_by_names(): # Length of get_countries_by_names dict and get_countries dict must be # the same, unless country names wouldn't be unique. @@ -88,3 +169,38 @@ def test_search_cities_name_contains_search_and_case_sensitive(): > len(gc.search_cities('London', 'name', case_sensitive=True, contains_search=False)) > 1 ) + + +def test_datasets_are_cached_per_instance(): + # Regression: _load_data used to return the parsed data without storing it, + # so every getter call re-read and re-parsed the JSON file from disk. + instance = GeonamesCache() + assert instance.cities is None + first = instance.get_cities() + assert instance.cities is not None + assert first is instance.get_cities() + + for getter, attribute in ( + (instance.get_countries, 'countries'), + (instance.get_admin1_codes, 'admin1'), + (instance.get_admin2_codes, 'admin2'), + (instance.get_timezones, 'timezones'), + (instance.get_us_states, 'us_states'), + (instance.get_us_counties, 'us_counties'), + ): + assert getattr(instance, attribute) is None + assert getter() is getter() + assert getattr(instance, attribute) is not None + + +def test_city_name_cache_is_not_shared_between_instances(): + # Regression: cities_by_names was a ClassVar keyed by name only, so an + # instance with a larger dataset got served another instance's results. + small = GeonamesCache(min_city_population=15000) + large = GeonamesCache(min_city_population=500) + + assert small.cities_by_names is not large.cities_by_names + + small_hits = small.get_cities_by_name('Springfield') + large_hits = large.get_cities_by_name('Springfield') + assert len(large_hits) > len(small_hits) From 8f7c4d6e1a8717c70be330feee8b2cb9c42b0366 Mon Sep 17 00:00:00 2001 From: Christiaan van Luik Date: Wed, 19 Aug 2026 10:14:11 +0200 Subject: [PATCH 3/5] Index city names instead of scanning once per name Closes #6. get_cities_by_name() scanned the whole cities dataset on every call for a name it had not seen, at 4 ms per name on the default dataset and 31 ms on the 500 population one. It now builds a name to records index on first call, so looking up many names costs one pass instead of one pass per name. Looking up 500 distinct names on the 500 population dataset went from 5.3 s to 0.08 s. The issue suggested generators. They do not help here: the results are memoised and a generator cannot be re-iterated, and the cost was the scan rather than building the result list. BREAKING: get_cities_by_name() returns a list of city records, where it returned a list of single-entry dictionaries keyed by geonameid before. Callers that did `list(d)[0]` or `next(iter(d))` to unwrap should read `city['geonameid']` instead, which the records already carry. Unknown names now return an empty list. The old shape was also what made an index expensive: building one single-key dictionary per city cost 68 MB on the 500 population dataset against 27 MB for plain references. The new get_cities_by_names() exposes the index itself, grouping every city record by name. Drops the cities_items attribute, which existed only to avoid re-listing the dataset's items on each scan and duplicated ~1.9 MB of tuples. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 9 +++++++++ README.md | 12 +++++++++--- geonamescache/__init__.py | 32 +++++++++++++++++++------------- tests/test_geonamescache.py | 29 ++++++++++++++++++++++++++++- 4 files changed, 65 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b102c66..4c62ee1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,15 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### Changed + +- **Breaking:** `get_cities_by_name()` now returns a list of city records, where it previously returned a list of single-entry dictionaries keyed by geonameid. Use `city['geonameid']` to get the id, which the records already carry. Unknown names return an empty list. +- `get_cities_by_name()` now builds an index of all city names on first call instead of scanning the dataset once per name. Looking up 500 distinct names on the 500 population dataset went from 5.3 s to 0.6 s, and the removed `cities_items` list no longer duplicates the dataset's items. + +### Added + +- Add `get_cities_by_names()` method returning all city records grouped by name, the index behind `get_cities_by_name()`. + ### Fixed - Fix dataset caching. `_load_data()` returned the parsed data without storing it, so every getter call re-read and re-parsed its JSON file from disk. Repeated `get_cities()` calls on the 500-population dataset took roughly 0.4 s each and now cost nothing after the first. diff --git a/README.md b/README.md index b73b08a..73685fb 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ Currently geonamescache provides the following methods, that return dictionaries * get\_countries\_by\_names() * get\_us\_states\_by\_names() * get\_cities\_by\_name(name) +* get\_cities\_by\_names() * get\_us\_counties() * get\_timezones() @@ -120,10 +121,15 @@ A dictionary keyed by geonameid **as a string**, holding 34078 cities at the def 'alternatenames': ['RTM', 'Ratehrdam', 'Roterdam', ...] } -City names are not unique, so `get_cities_by_name()` returns a list of single-entry dictionaries rather than one record. There is a Rotterdam in both the Netherlands and the US state of New York: +City names are not unique, so `get_cities_by_name()` returns a list of records. There is a Rotterdam in both the Netherlands and the US state of New York: - >>> [list(d) for d in gc.get_cities_by_name('Rotterdam')] - [['2747891'], ['5134453']] + >>> [(c['geonameid'], c['countrycode']) for c in gc.get_cities_by_name('Rotterdam')] + [(2747891, 'NL'), (5134453, 'US')] + +Unknown names give an empty list. The first call builds an index of every city name, so looking up many names costs one pass over the dataset instead of one pass per name. `get_cities_by_names()` returns that whole index, a dictionary mapping each name to its list of records: + + >>> len(gc.get_cities_by_names()) + 32215 `search_cities()` returns a flat list of city records instead. It searches `alternatenames` by default, so it matches places whose *other* names contain the query, here the Rotterdam district of Hoogvliet: diff --git a/geonamescache/__init__.py b/geonamescache/__init__.py index ab720ef..f70a111 100644 --- a/geonamescache/__init__.py +++ b/geonamescache/__init__.py @@ -38,16 +38,14 @@ class GeonamesCache: continents: dict[ContinentCode, Continent] | None = None countries: dict[ISOStr, Country] | None = None cities: dict[GeoNameIdStr, City] | None = None - cities_items: list[tuple[GeoNameIdStr, City]] | None = None timezones: dict[TimeZoneIdStr, TimeZoneInfo] | None = None us_counties: list[USCounty] | None = None us_states: dict[USStateCode, USState] | None = None def __init__(self, min_city_population: int = 15000): self.min_city_population = min_city_population - # Per instance, because the cache key is the city name alone while the - # results depend on min_city_population. - self.cities_by_names: dict[str, list[dict[GeoNameIdStr, City]]] = {} + # Per instance, because it indexes one particular cities dataset. + self.cities_by_names: dict[str, list[City]] | None = None def get_dataset_by_key(self, dataset: dict[Any, TDict], key: str) -> dict[Any, TDict]: return {d[key]: d for c, d in list(dataset.items())} @@ -129,17 +127,25 @@ def get_cities(self) -> dict[GeoNameIdStr, City]: self.cities = self._load_data(f'cities{self.min_city_population}.json') return self.cities - def get_cities_by_name(self, name: str) -> list[dict[GeoNameIdStr, City]]: - """Get a list of city dictionaries with the given name. + def get_cities_by_names(self) -> dict[str, list[City]]: + """Get city records grouped by name. - City names cannot be used as keys, as they are not unique. + City names are not unique, so each name maps to a list of records. """ - - if name not in self.cities_by_names: - if self.cities_items is None: - self.cities_items = list(self.get_cities().items()) - self.cities_by_names[name] = [{gid: city} for gid, city in self.cities_items if city['name'] == name] - return self.cities_by_names[name] + if self.cities_by_names is None: + index: dict[str, list[City]] = {} + for city in self.get_cities().values(): + index.setdefault(city['name'], []).append(city) + self.cities_by_names = index + return self.cities_by_names + + def get_cities_by_name(self, name: str) -> list[City]: + """Get the city records with the given name, empty list if there are none. + + Builds an index of all city names on first call, so looking up many + names costs one pass over the dataset rather than one pass per name. + """ + return self.get_cities_by_names().get(name, []) def get_us_counties(self) -> list[USCounty]: if self.us_counties is None: diff --git a/tests/test_geonamescache.py b/tests/test_geonamescache.py index e122ee2..80fd03c 100644 --- a/tests/test_geonamescache.py +++ b/tests/test_geonamescache.py @@ -120,6 +120,33 @@ def test_get_cities_by_name_madrid(): assert len(gc.get_cities_by_name('Madrid')) > 1 +def test_get_cities_by_name_returns_city_records(): + rotterdams = gc.get_cities_by_name('Rotterdam') + assert 2 == len(rotterdams) + # Records are returned directly, not wrapped in single-key dictionaries. + assert ['NL', 'US'] == sorted(city['countrycode'] for city in rotterdams) + assert all('Rotterdam' == city['name'] for city in rotterdams) + + # The records are the same objects held by get_cities(), not copies. + assert gc.get_cities()['2747891'] in rotterdams + + +def test_get_cities_by_name_unknown(): + assert [] == gc.get_cities_by_name('Nonexistent Place') + + +def test_get_cities_by_names_index(): + index = gc.get_cities_by_names() + assert index is gc.get_cities_by_names() + + # Every city must appear under its own name, and nothing may be lost. + cities = gc.get_cities() + assert sum(len(records) for records in index.values()) == len(cities) + assert len(index) < len(cities) # names are not unique + for city in cities.values(): + assert city in index[city['name']] + + def test_cities_in_us_states(): cities = gc.get_cities() for gid, name, us_state in (('4164138', 'Miami', 'FL'), ('4525353', 'Springfield', 'OH')): @@ -199,7 +226,7 @@ def test_city_name_cache_is_not_shared_between_instances(): small = GeonamesCache(min_city_population=15000) large = GeonamesCache(min_city_population=500) - assert small.cities_by_names is not large.cities_by_names + assert small.get_cities_by_names() is not large.get_cities_by_names() small_hits = small.get_cities_by_name('Springfield') large_hits = large.get_cities_by_name('Springfield') From f194b67bf5d8b16154593f6b48b32021a4ce0d8f Mon Sep 17 00:00:00 2001 From: Christiaan van Luik Date: Wed, 19 Aug 2026 10:24:25 +0200 Subject: [PATCH 4/5] Store bundled datasets gzipped Cuts the installed size of the package from 203 MB to 36 MB. Load time is unchanged: the first get_cities() call on the 500 population dataset takes 0.31 s either way, as the saved I/O offsets the decompression. Wheel and sdist size are unchanged at ~35 MB, because those were already deflate compressed. This is a disk win rather than a bandwidth one. The bin/ scripts keep writing plain JSON into datasets/, and the new bin/compress_data.py gzips it into geonamescache/data/ as the last step of `make json`, replacing the plain mv. It sets mtime=0 so identical input produces identical output and rebuilds don't churn the package data. _load_data() now takes a dataset name rather than a file name and appends the .json.gz suffix itself. Verified end to end by building a wheel and using it from a clean virtualenv. A test asserts no stale plain .json is left in the data directory, since it would be shipped and silently ignored. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 2 ++ Makefile | 2 +- README.md | 4 ++-- bin/compress_data.py | 24 ++++++++++++++++++++++++ geonamescache/__init__.py | 24 +++++++++++++----------- tests/test_data.py | 11 +++++++++++ 6 files changed, 53 insertions(+), 14 deletions(-) create mode 100755 bin/compress_data.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c62ee1..41b541e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed +- Store the bundled datasets gzipped. This cuts the installed size of the package from 203 MB to 36 MB at no cost in load time, as the saved I/O offsets the decompression. Wheel and sdist size are unchanged, since those were already compressed. Building the data now ends with `./bin/compress_data.py` instead of moving the JSON files into place. + - **Breaking:** `get_cities_by_name()` now returns a list of city records, where it previously returned a list of single-entry dictionaries keyed by geonameid. Use `city['geonameid']` to get the id, which the records already carry. Unknown names return an empty list. - `get_cities_by_name()` now builds an index of all city names on first call instead of scanning the dataset once per name. Looking up 500 distinct names on the 500 population dataset went from 5.3 s to 0.6 s, and the removed `cities_items` list no longer duplicates the dataset's items. diff --git a/Makefile b/Makefile index f7dfc62..b175164 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ json: ./bin/us_counties.py ./bin/us_states.py ./bin/timezones.py - mv datasets/*.json geonamescache/data/ + ./bin/compress_data.py clean: clean-build clean-py clean-dev clean-datasets diff --git a/README.md b/README.md index 73685fb..2fc4bbf 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ A simple example: gc = geonamescache.GeonamesCache() print(gc.get_countries()) -Each `GeonamesCache` instance caches every dataset it loads, so keep one instance around rather than creating a new one per lookup. +The datasets are bundled gzipped and parsed on first use, so the installed package is about 36 MB. Each `GeonamesCache` instance caches every dataset it loads, so keep one instance around rather than creating a new one per lookup. ## Settings @@ -242,4 +242,4 @@ The mappers module provides function(s) to map data properties. Currently you ca ## Contributing -Please write test(s) for any new feature. If you wish to build the data from scratch, run `make dl` and `make json`. +Please write test(s) for any new feature. If you wish to build the data from scratch, run `make dl` and `make json`. The `bin/` scripts write plain JSON into `datasets/`, and `bin/compress_data.py` gzips it into `geonamescache/data/` as the last step of `make json`. diff --git a/bin/compress_data.py b/bin/compress_data.py new file mode 100755 index 0000000..ef7d219 --- /dev/null +++ b/bin/compress_data.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +"""Gzip the built JSON files into the package data directory. + +Storing the data gzipped cuts the installed size of the package by about a +factor of six without costing load time, as the saved I/O offsets the +decompression. Keeping this a separate step means the bin/ scripts stay +plain JSON writers. +""" +import gzip +import shutil +from pathlib import Path + +p_src = Path('datasets') +p_dst = Path('geonamescache', 'data') +p_dst.mkdir(parents=True, exist_ok=True) + +for p_json in sorted(p_src.glob('*.json')): + p_gz = p_dst.joinpath(p_json.name + '.gz') + # mtime=0 keeps the output byte identical for identical input, so rebuilds + # don't churn the package data. + with p_json.open('rb') as f_in, gzip.GzipFile(p_gz, 'wb', compresslevel=9, mtime=0) as f_out: + shutil.copyfileobj(f_in, f_out) + print(f'{p_json.name}: {p_json.stat().st_size / 1e6:.1f} MB -> {p_gz.stat().st_size / 1e6:.1f} MB') + p_json.unlink() diff --git a/geonamescache/__init__.py b/geonamescache/__init__.py index f70a111..aa32f98 100644 --- a/geonamescache/__init__.py +++ b/geonamescache/__init__.py @@ -4,6 +4,7 @@ __license__ = 'MIT' +import gzip import json import os from collections.abc import Mapping @@ -52,24 +53,24 @@ def get_dataset_by_key(self, dataset: dict[Any, TDict], key: str) -> dict[Any, T def get_continents(self) -> dict[ContinentCode, Continent]: if self.continents is None: - self.continents = self._load_data('continents.json') + self.continents = self._load_data('continents') return self.continents def get_countries(self) -> dict[ISOStr, Country]: if self.countries is None: - self.countries = self._load_data('countries.json') + self.countries = self._load_data('countries') return self.countries def get_admin1_codes(self) -> dict[Admin1CodeStr, Admin1]: """Get first-level administrative divisions keyed by ., e. g. US.CA.""" if self.admin1 is None: - self.admin1 = self._load_data('admin1.json') + self.admin1 = self._load_data('admin1') return self.admin1 def get_admin2_codes(self) -> dict[Admin2CodeStr, Admin2]: """Get second-level administrative divisions keyed by .., e. g. NL.11.0599.""" if self.admin2 is None: - self.admin2 = self._load_data('admin2.json') + self.admin2 = self._load_data('admin2') return self.admin2 def get_admin1_by_city(self, city: City) -> Admin1 | None: @@ -95,7 +96,7 @@ def get_admin2_by_city(self, city: City) -> Admin2 | None: def get_timezones(self) -> dict[TimeZoneIdStr, TimeZoneInfo]: """Get time zones keyed by IANA time zone id, e. g. Europe/Amsterdam.""" if self.timezones is None: - self.timezones = self._load_data('timezones.json') + self.timezones = self._load_data('timezones') return self.timezones def get_timezones_by_country(self, countrycode: str) -> list[TimeZoneInfo]: @@ -112,7 +113,7 @@ def get_timezones_by_country(self, countrycode: str) -> list[TimeZoneInfo]: def get_us_states(self) -> dict[USStateCode, USState]: if self.us_states is None: - self.us_states = self._load_data('us_states.json') + self.us_states = self._load_data('us_states') return self.us_states def get_countries_by_names(self) -> dict[str, Country]: @@ -124,7 +125,7 @@ def get_us_states_by_names(self) -> dict[USStateName, USState]: def get_cities(self) -> dict[GeoNameIdStr, City]: """Get a dictionary of cities keyed by geonameid.""" if self.cities is None: - self.cities = self._load_data(f'cities{self.min_city_population}.json') + self.cities = self._load_data(f'cities{self.min_city_population}') return self.cities def get_cities_by_names(self) -> dict[str, list[City]]: @@ -149,7 +150,7 @@ def get_cities_by_name(self, name: str) -> list[City]: def get_us_counties(self) -> list[USCounty]: if self.us_counties is None: - self.us_counties = self._load_data('us_counties.json') + self.us_counties = self._load_data('us_counties') return self.us_counties def search_cities( @@ -182,7 +183,8 @@ def search_cities( return results @staticmethod - def _load_data(datafile: str) -> Any: - """Read and parse a bundled data file. Callers are responsible for caching.""" - with open(os.path.join(os.path.dirname(__file__), 'data', datafile), encoding='utf-8') as f: + def _load_data(dataset: str) -> Any: + """Read and parse a bundled dataset. Callers are responsible for caching.""" + path = os.path.join(os.path.dirname(__file__), 'data', dataset + '.json.gz') + with gzip.open(path, 'rt', encoding='utf-8') as f: return json.load(f) diff --git a/tests/test_data.py b/tests/test_data.py index ed15d9a..e8eb147 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -1,3 +1,6 @@ +from pathlib import Path + +import geonamescache from geonamescache import GeonamesCache gc = GeonamesCache() @@ -66,3 +69,11 @@ def test_us_states(): for code in ['XX', 'OO']: assert code not in us_states + + +def test_data_files_are_gzipped(): + # The bundled datasets are stored gzipped to keep the installed package + # small. A stale plain .json alongside them would be shipped and ignored. + data_dir = Path(geonamescache.__file__).parent / 'data' + assert sorted(p.name for p in data_dir.glob('*.json')) == [] + assert len(list(data_dir.glob('*.json.gz'))) > 5 From f5f9eb0f99646ef3e04f825e57171388562a5416 Mon Sep 17 00:00:00 2001 From: Christiaan van Luik Date: Wed, 19 Aug 2026 10:28:59 +0200 Subject: [PATCH 5/5] Expose featurecode on city records The GeoNames feature code distinguishes a capital (PPLC) or an administrative seat (PPLA through PPLA5) from an ordinary populated place (PPL). The cities datasets were already carrying it in column 8 and the build discarded it. It also explains a discrepancy that is otherwise invisible: cities15000.txt contains 45 places with a population below 15000, all of them PPLC or PPLG, because GeoNames includes seats of government regardless of size. The rule `population > threshold or featurecode in seats` reproduces each cities file from cities500.txt exactly, with the seat codes accumulating as the threshold drops. featureclass is deliberately not stored, as it is P for every record in these datasets and so carries no information. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 1 + README.md | 6 ++++++ bin/cities.py | 1 + geonamescache/types.py | 5 ++++- tests/test_geonamescache.py | 19 +++++++++++++++++++ 5 files changed, 31 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41b541e..0866ccf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added +- Add `featurecode` to city records, the GeoNames feature code that distinguishes a capital (`PPLC`) or administrative seat (`PPLA` through `PPLA5`) from an ordinary populated place (`PPL`). It is also searchable via `search_cities(attribute='featurecode')`. The feature class is always `P` in these datasets and is not stored. - Add `get_cities_by_names()` method returning all city records grouped by name, the index behind `get_cities_by_name()`. ### Fixed diff --git a/README.md b/README.md index 2fc4bbf..e040e1d 100644 --- a/README.md +++ b/README.md @@ -118,9 +118,15 @@ A dictionary keyed by geonameid **as a string**, holding 34078 cities at the def 'timezone': 'Europe/Amsterdam', 'admin1code': '11', 'admin2code': '0599', + 'featurecode': 'PPL', 'alternatenames': ['RTM', 'Ratehrdam', 'Roterdam', ...] } +`featurecode` is the GeoNames [feature code](http://www.geonames.org/export/codes.html), which distinguishes a capital (`PPLC`) or an administrative seat (`PPLA` through `PPLA5`) from an ordinary populated place (`PPL`). It is what lets the datasets include capitals below their population threshold, such as Nuuk and Tórshavn. The feature *class* is always `P` in these datasets, so it is not stored. You can search on it: + + >>> len(gc.search_cities('PPLC', attribute='featurecode', contains_search=False)) + 241 + City names are not unique, so `get_cities_by_name()` returns a list of records. There is a Rotterdam in both the Netherlands and the US state of New York: >>> [(c['geonameid'], c['countrycode']) for c in gc.get_cities_by_name('Rotterdam')] diff --git a/bin/cities.py b/bin/cities.py index ccad13e..d8ac56c 100755 --- a/bin/cities.py +++ b/bin/cities.py @@ -47,6 +47,7 @@ 'timezone': timezone, 'admin1code': admin1code, 'admin2code': admin2code, + 'featurecode': featurecode, 'alternatenames': alternatenames.split(','), } diff --git a/geonamescache/types.py b/geonamescache/types.py index f40cf92..d3a9d62 100644 --- a/geonamescache/types.py +++ b/geonamescache/types.py @@ -118,7 +118,9 @@ "West Virginia", "Wyoming", ] -CitySearchAttribute = Literal["alternatenames", "admin1code", "admin2code", "countrycode", "name", "timezone"] +CitySearchAttribute = Literal[ + "alternatenames", "admin1code", "admin2code", "countrycode", "featurecode", "name", "timezone" +] class TimeZone(TypedDict): @@ -195,6 +197,7 @@ class City(TypedDict): admin1code: str admin2code: str countrycode: str + featurecode: str geonameid: int latitude: float longitude: float diff --git a/tests/test_geonamescache.py b/tests/test_geonamescache.py index 80fd03c..29b2e4c 100644 --- a/tests/test_geonamescache.py +++ b/tests/test_geonamescache.py @@ -104,6 +104,25 @@ def test_city_timezone_is_in_timezones(): assert all(city['timezone'] in timezones for city in gc.get_cities().values()) +def test_city_featurecode(): + cities = gc.get_cities() + assert 'PPL' == cities['2747891']['featurecode'] + # Feature codes are never blank in these datasets. + assert all(city['featurecode'] for city in cities.values()) + + # PPLC marks the capital of a political entity, which is how the datasets + # include capitals that fall below their population threshold. + capitals = {city['name'] for city in cities.values() if city['featurecode'] == 'PPLC'} + for capital in ('Nuuk', 'Belmopan', 'Madrid', 'Washington'): + assert capital in capitals + + +def test_search_cities_by_featurecode(): + seats = gc.search_cities('PPLG', attribute='featurecode', contains_search=False) + assert seats + assert all('PPLG' == city['featurecode'] for city in seats) + + def test_get_countries_by_names(): # Length of get_countries_by_names dict and get_countries dict must be # the same, unless country names wouldn't be unique.