diff options
Diffstat (limited to 'build_renfe')
| -rw-r--r-- | build_renfe/LICENCE | 287 | ||||
| -rw-r--r-- | build_renfe/build_static_feed.py | 91 | ||||
| -rw-r--r-- | build_renfe/stop_overrides.json | 6 |
3 files changed, 79 insertions, 305 deletions
diff --git a/build_renfe/LICENCE b/build_renfe/LICENCE deleted file mode 100644 index 4153cd3..0000000 --- a/build_renfe/LICENCE +++ /dev/null @@ -1,287 +0,0 @@ - EUROPEAN UNION PUBLIC LICENCE v. 1.2 - EUPL © the European Union 2007, 2016 - -This European Union Public Licence (the ‘EUPL’) applies to the Work (as defined -below) which is provided under the terms of this Licence. Any use of the Work, -other than as authorised under this Licence is prohibited (to the extent such -use is covered by a right of the copyright holder of the Work). - -The Work is provided under the terms of this Licence when the Licensor (as -defined below) has placed the following notice immediately following the -copyright notice for the Work: - - Licensed under the EUPL - -or has expressed by any other means his willingness to license under the EUPL. - -1. Definitions - -In this Licence, the following terms have the following meaning: - -- ‘The Licence’: this Licence. - -- ‘The Original Work’: the work or software distributed or communicated by the - Licensor under this Licence, available as Source Code and also as Executable - Code as the case may be. - -- ‘Derivative Works’: the works or software that could be created by the - Licensee, based upon the Original Work or modifications thereof. This Licence - does not define the extent of modification or dependence on the Original Work - required in order to classify a work as a Derivative Work; this extent is - determined by copyright law applicable in the country mentioned in Article 15. - -- ‘The Work’: the Original Work or its Derivative Works. - -- ‘The Source Code’: the human-readable form of the Work which is the most - convenient for people to study and modify. - -- ‘The Executable Code’: any code which has generally been compiled and which is - meant to be interpreted by a computer as a program. - -- ‘The Licensor’: the natural or legal person that distributes or communicates - the Work under the Licence. - -- ‘Contributor(s)’: any natural or legal person who modifies the Work under the - Licence, or otherwise contributes to the creation of a Derivative Work. - -- ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of - the Work under the terms of the Licence. - -- ‘Distribution’ or ‘Communication’: any act of selling, giving, lending, - renting, distributing, communicating, transmitting, or otherwise making - available, online or offline, copies of the Work or providing access to its - essential functionalities at the disposal of any other natural or legal - person. - -2. Scope of the rights granted by the Licence - -The Licensor hereby grants You a worldwide, royalty-free, non-exclusive, -sublicensable licence to do the following, for the duration of copyright vested -in the Original Work: - -- use the Work in any circumstance and for all usage, -- reproduce the Work, -- modify the Work, and make Derivative Works based upon the Work, -- communicate to the public, including the right to make available or display - the Work or copies thereof to the public and perform publicly, as the case may - be, the Work, -- distribute the Work or copies thereof, -- lend and rent the Work or copies thereof, -- sublicense rights in the Work or copies thereof. - -Those rights can be exercised on any media, supports and formats, whether now -known or later invented, as far as the applicable law permits so. - -In the countries where moral rights apply, the Licensor waives his right to -exercise his moral right to the extent allowed by law in order to make effective -the licence of the economic rights here above listed. - -The Licensor grants to the Licensee royalty-free, non-exclusive usage rights to -any patents held by the Licensor, to the extent necessary to make use of the -rights granted on the Work under this Licence. - -3. Communication of the Source Code - -The Licensor may provide the Work either in its Source Code form, or as -Executable Code. If the Work is provided as Executable Code, the Licensor -provides in addition a machine-readable copy of the Source Code of the Work -along with each copy of the Work that the Licensor distributes or indicates, in -a notice following the copyright notice attached to the Work, a repository where -the Source Code is easily and freely accessible for as long as the Licensor -continues to distribute or communicate the Work. - -4. Limitations on copyright - -Nothing in this Licence is intended to deprive the Licensee of the benefits from -any exception or limitation to the exclusive rights of the rights owners in the -Work, of the exhaustion of those rights or of other applicable limitations -thereto. - -5. Obligations of the Licensee - -The grant of the rights mentioned above is subject to some restrictions and -obligations imposed on the Licensee. Those obligations are the following: - -Attribution right: The Licensee shall keep intact all copyright, patent or -trademarks notices and all notices that refer to the Licence and to the -disclaimer of warranties. The Licensee must include a copy of such notices and a -copy of the Licence with every copy of the Work he/she distributes or -communicates. The Licensee must cause any Derivative Work to carry prominent -notices stating that the Work has been modified and the date of modification. - -Copyleft clause: If the Licensee distributes or communicates copies of the -Original Works or Derivative Works, this Distribution or Communication will be -done under the terms of this Licence or of a later version of this Licence -unless the Original Work is expressly distributed only under this version of the -Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee -(becoming Licensor) cannot offer or impose any additional terms or conditions on -the Work or Derivative Work that alter or restrict the terms of the Licence. - -Compatibility clause: If the Licensee Distributes or Communicates Derivative -Works or copies thereof based upon both the Work and another work licensed under -a Compatible Licence, this Distribution or Communication can be done under the -terms of this Compatible Licence. For the sake of this clause, ‘Compatible -Licence’ refers to the licences listed in the appendix attached to this Licence. -Should the Licensee's obligations under the Compatible Licence conflict with -his/her obligations under this Licence, the obligations of the Compatible -Licence shall prevail. - -Provision of Source Code: When distributing or communicating copies of the Work, -the Licensee will provide a machine-readable copy of the Source Code or indicate -a repository where this Source will be easily and freely available for as long -as the Licensee continues to distribute or communicate the Work. - -Legal Protection: This Licence does not grant permission to use the trade names, -trademarks, service marks, or names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the copyright notice. - -6. Chain of Authorship - -The original Licensor warrants that the copyright in the Original Work granted -hereunder is owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each Contributor warrants that the copyright in the modifications he/she brings -to the Work are owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each time You accept the Licence, the original Licensor and subsequent -Contributors grant You a licence to their contributions to the Work, under the -terms of this Licence. - -7. Disclaimer of Warranty - -The Work is a work in progress, which is continuously improved by numerous -Contributors. It is not a finished work and may therefore contain defects or -‘bugs’ inherent to this type of development. - -For the above reason, the Work is provided under the Licence on an ‘as is’ basis -and without warranties of any kind concerning the Work, including without -limitation merchantability, fitness for a particular purpose, absence of defects -or errors, accuracy, non-infringement of intellectual property rights other than -copyright as stated in Article 6 of this Licence. - -This disclaimer of warranty is an essential part of the Licence and a condition -for the grant of any rights to the Work. - -8. Disclaimer of Liability - -Except in the cases of wilful misconduct or damages directly caused to natural -persons, the Licensor will in no event be liable for any direct or indirect, -material or moral, damages of any kind, arising out of the Licence or of the use -of the Work, including without limitation, damages for loss of goodwill, work -stoppage, computer failure or malfunction, loss of data or any commercial -damage, even if the Licensor has been advised of the possibility of such damage. -However, the Licensor will be liable under statutory product liability laws as -far such laws apply to the Work. - -9. Additional agreements - -While distributing the Work, You may choose to conclude an additional agreement, -defining obligations or services consistent with this Licence. However, if -accepting obligations, You may act only on your own behalf and on your sole -responsibility, not on behalf of the original Licensor or any other Contributor, -and only if You agree to indemnify, defend, and hold each Contributor harmless -for any liability incurred by, or claims asserted against such Contributor by -the fact You have accepted any warranty or additional liability. - -10. Acceptance of the Licence - -The provisions of this Licence can be accepted by clicking on an icon ‘I agree’ -placed under the bottom of a window displaying the text of this Licence or by -affirming consent in any other similar way, in accordance with the rules of -applicable law. Clicking on that icon indicates your clear and irrevocable -acceptance of this Licence and all of its terms and conditions. - -Similarly, you irrevocably accept this Licence and all of its terms and -conditions by exercising any rights granted to You by Article 2 of this Licence, -such as the use of the Work, the creation by You of a Derivative Work or the -Distribution or Communication by You of the Work or copies thereof. - -11. Information to the public - -In case of any Distribution or Communication of the Work by means of electronic -communication by You (for example, by offering to download the Work from a -remote location) the distribution channel or media (for example, a website) must -at least provide to the public the information requested by the applicable law -regarding the Licensor, the Licence and the way it may be accessible, concluded, -stored and reproduced by the Licensee. - -12. Termination of the Licence - -The Licence and the rights granted hereunder will terminate automatically upon -any breach by the Licensee of the terms of the Licence. - -Such a termination will not terminate the licences of any person who has -received the Work from the Licensee under the Licence, provided such persons -remain in full compliance with the Licence. - -13. Miscellaneous - -Without prejudice of Article 9 above, the Licence represents the complete -agreement between the Parties as to the Work. - -If any provision of the Licence is invalid or unenforceable under applicable -law, this will not affect the validity or enforceability of the Licence as a -whole. Such provision will be construed or reformed so as necessary to make it -valid and enforceable. - -The European Commission may publish other linguistic versions or new versions of -this Licence or updated versions of the Appendix, so far this is required and -reasonable, without reducing the scope of the rights granted by the Licence. New -versions of the Licence will be published with a unique version number. - -All linguistic versions of this Licence, approved by the European Commission, -have identical value. Parties can take advantage of the linguistic version of -their choice. - -14. Jurisdiction - -Without prejudice to specific agreement between parties, - -- any litigation resulting from the interpretation of this License, arising - between the European Union institutions, bodies, offices or agencies, as a - Licensor, and any Licensee, will be subject to the jurisdiction of the Court - of Justice of the European Union, as laid down in article 272 of the Treaty on - the Functioning of the European Union, - -- any litigation arising between other parties and resulting from the - interpretation of this License, will be subject to the exclusive jurisdiction - of the competent court where the Licensor resides or conducts its primary - business. - -15. Applicable Law - -Without prejudice to specific agreement between parties, - -- this Licence shall be governed by the law of the European Union Member State - where the Licensor has his seat, resides or has his registered office, - -- this licence shall be governed by Belgian law if the Licensor has no seat, - residence or registered office inside a European Union Member State. - -Appendix - -‘Compatible Licences’ according to Article 5 EUPL are: - -- GNU General Public License (GPL) v. 2, v. 3 -- GNU Affero General Public License (AGPL) v. 3 -- Open Software License (OSL) v. 2.1, v. 3.0 -- Eclipse Public License (EPL) v. 1.0 -- CeCILL v. 2.0, v. 2.1 -- Mozilla Public Licence (MPL) v. 2 -- GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 -- Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for - works other than software -- European Union Public Licence (EUPL) v. 1.1, v. 1.2 -- Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or Strong - Reciprocity (LiLiQ-R+). - -The European Commission may update this Appendix to later versions of the above -licences without producing a new version of the EUPL, as long as they provide -the rights granted in Article 2 of this Licence and protect the covered Source -Code from exclusive appropriation. - -All other changes or additions to this Appendix require the production of a new -EUPL version. diff --git a/build_renfe/build_static_feed.py b/build_renfe/build_static_feed.py index a60360f..eb247a9 100644 --- a/build_renfe/build_static_feed.py +++ b/build_renfe/build_static_feed.py @@ -1,6 +1,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ +# "pandas", # "requests", # "tqdm", # ] @@ -8,6 +9,7 @@ from argparse import ArgumentParser import csv +import io import json import logging import os @@ -15,6 +17,8 @@ import shutil import tempfile import zipfile +import pandas as pd + import requests from tqdm import tqdm @@ -24,8 +28,7 @@ BOUNDS = {"SOUTH": 41.820455, "NORTH": 43.937462, "WEST": -9.437256, "EAST": -6. FEEDS = { "general": "1098", - "cercanias": "1130", - "feve": "1131" + "cercanias": "1130" } @@ -189,6 +192,11 @@ if __name__ == "__main__": help="Enable debug logging", action="store_true" ) + parser.add_argument( + "--merge", + help="Merge the generated feeds into a single GTFS ZIP file instead of separate ones for each feed", + action="store_true" + ) args = parser.parse_args() @@ -310,21 +318,21 @@ if __name__ == "__main__": os.path.join(INPUT_GTFS_PATH, "routes.txt"), "route_id", route_ids ) - if feed == "feve": - feve_c1_route_ids = ["46T0001C1", "46T0002C1"] - new_route_id = "FEVE_C1" + if feed == "cercanias": + cercanias_c1_route_ids = ["46T0001C1", "46T0002C1"] + new_route_id = "FERROL_C1" # Find agency_id and a template route template_route = routes_in_trips[0] if routes_in_trips else {} agency_id = "1" for r in routes_in_trips: - if r["route_id"].strip() in feve_c1_route_ids: + if r["route_id"].strip() in cercanias_c1_route_ids: agency_id = r.get("agency_id", "1") template_route = r break # Filter out old routes - routes_in_trips = [r for r in routes_in_trips if r["route_id"].strip() not in feve_c1_route_ids] + routes_in_trips = [r for r in routes_in_trips if r["route_id"].strip() not in cercanias_c1_route_ids] # Add new route new_route = template_route.copy() @@ -377,13 +385,13 @@ if __name__ == "__main__": trips_in_galicia = get_rows_by_ids(TRIPS_FILE, "trip_id", trip_ids) - if feed == "feve": - feve_c1_route_ids = ["46T0001C1", "46T0002C1"] - new_route_id = "FEVE_C1" + if feed == "cercanias": + cercanias_c1_route_ids = ["46T0001C1", "46T0002C1"] + new_route_id = "FERROL_C1" for tig in trips_in_galicia: - if tig["route_id"].strip() in feve_c1_route_ids: - tig["route_id"] = new_route_id + if tig["route_id"].strip() in cercanias_c1_route_ids: tig["direction_id"] = "1" if tig["route_id"].strip()[6] == "2" else "0" + tig["route_id"] = new_route_id stops_by_id = {stop["stop_id"]: stop for stop in stops_in_trips} @@ -562,3 +570,62 @@ if __name__ == "__main__": os.remove(INPUT_GTFS_ZIP) shutil.rmtree(INPUT_GTFS_PATH) shutil.rmtree(OUTPUT_GTFS_PATH) + + if args.merge: + # Columns to keep for each GTFS file when merging. + # Files not listed here keep all columns present in the data. + MERGE_KEEP_COLS: dict[str, list[str]] = { + "agency.txt": ["agency_id", "agency_name", "agency_url", "agency_timezone", "agency_lang"], + "stops.txt": ["stop_id", "stop_code", "stop_name", "stop_lat", "stop_lon", "wheelchair_boarding"], + "routes.txt": ["route_id", "agency_id", "route_short_name", "route_long_name", "route_type", "route_color", "route_text_color"], + "trips.txt": ["route_id", "service_id", "trip_id", "trip_headsign", "direction_id", "shape_id", "wheelchair_accessible"], + "stop_times.txt": ["trip_id", "arrival_time", "departure_time", "stop_id", "stop_sequence", "pickup_type", "drop_off_type"], + } + # Default values to fill for columns that are missing or NaN after concat. + MERGE_FILL_DEFAULTS: dict[str, dict[str, str]] = { + "routes.txt": {"agency_id": "1071VC"}, + "trips.txt": {"direction_id": "0", "shape_id": "", "wheelchair_accessible": ""}, + "stop_times.txt": {"pickup_type": "0", "drop_off_type": "0"}, + } + # Deduplicate rows by this column, keeping the first occurrence. + MERGE_DEDUP_KEY: dict[str, str] = { + "stops.txt": "stop_id", + } + + merged_zip_path = os.path.join(os.path.dirname(__file__), "gtfs_renfe_galicia_merged.zip") + feed_zip_paths = [os.path.join(os.path.dirname(__file__), f"gtfs_renfe_galicia_{feed}.zip") for feed in FEEDS.keys()] + + frames: dict[str, list[pd.DataFrame]] = {} + for feed_zip_path in feed_zip_paths: + with zipfile.ZipFile(feed_zip_path, "r") as feed_zip: + for filename in feed_zip.namelist(): + with feed_zip.open(filename) as f: + df = pd.read_csv(f, dtype=str, encoding="utf-8") + df.columns = df.columns.str.strip() + df = df.apply(lambda col: col.str.strip() if col.dtype == object else col) + frames.setdefault(filename, []).append(df) + + with zipfile.ZipFile(merged_zip_path, "w", zipfile.ZIP_DEFLATED) as merged_zip: + for filename, dfs in frames.items(): + merged = pd.concat(dfs, ignore_index=True) + + keep = MERGE_KEEP_COLS.get(filename) + defaults = MERGE_FILL_DEFAULTS.get(filename, {}) + if keep is not None: + for col in keep: + if col not in merged.columns: + merged[col] = defaults.get(col, "") + for col, val in defaults.items(): + if col in merged.columns: + merged[col] = merged[col].fillna(val) + merged = merged[keep] + + dedup_key = MERGE_DEDUP_KEY.get(filename) + if dedup_key: + merged = merged.drop_duplicates(subset=[dedup_key], keep="first") + + buf = io.StringIO() + merged.to_csv(buf, index=False) + merged_zip.writestr(filename, buf.getvalue()) + + logging.info(f"Feeds merged successfully into {merged_zip_path}.")
\ No newline at end of file diff --git a/build_renfe/stop_overrides.json b/build_renfe/stop_overrides.json index c2298fa..5f4b5b8 100644 --- a/build_renfe/stop_overrides.json +++ b/build_renfe/stop_overrides.json @@ -73,12 +73,6 @@ "stop_lon": -8.230795701069612 }, { - "stop_id": "21010", - "feed_id": "feve", - "stop_lat": 43.48826050175589, - "stop_lon": -8.231122670037813 - }, - { "stop_id": "99117", "stop_name": "Ourense Turístico", "_delete": true |