From c2a17e60cad8f2f0f68d7a169e6f006bc4fb6b0d Mon Sep 17 00:00:00 2001 From: Jonathan Harris Date: Thu, 14 Sep 2017 19:49:38 +0100 Subject: [PATCH] Collate commodities indexed by id and including symbolic name Use collated info in export to older tools. --- EDMC.py | 11 ++- EDMarketConnector.wxs | 4 + collate.py | 20 ++--- commodity.csv | 203 ++++++++++++++++++++++++++++++++++++++++++ companion.py | 18 +++- setup.py | 3 +- 6 files changed, 239 insertions(+), 20 deletions(-) create mode 100644 commodity.csv diff --git a/EDMC.py b/EDMC.py index 87b72bca..50604d51 100755 --- a/EDMC.py +++ b/EDMC.py @@ -177,17 +177,16 @@ try: # Finally - the data looks sane and we're docked at a station - # Fixup anomalies in the commodity data - fixed = companion.fixup(data) - if args.j: # Collate from JSON dump - collate.addcommodities(fixed) - collate.addmodules(fixed) - collate.addships(fixed) + collate.addcommodities(data) + collate.addmodules(data) + collate.addships(data) if args.m: if data['lastStarport'].get('commodities'): + # Fixup anomalies in the commodity data + fixed = companion.fixup(data) commodity.export(fixed, COMMODITY_DEFAULT, args.m) else: sys.stderr.write("Station doesn't have a market\n") diff --git a/EDMarketConnector.wxs b/EDMarketConnector.wxs index 4981201a..7681a33a 100644 --- a/EDMarketConnector.wxs +++ b/EDMarketConnector.wxs @@ -123,6 +123,9 @@ + + + @@ -455,6 +458,7 @@ + diff --git a/collate.py b/collate.py index e0a0c1f0..d2a8ce57 100755 --- a/collate.py +++ b/collate.py @@ -27,19 +27,20 @@ def addcommodities(data): with open(commodityfile) as csvfile: reader = csv.DictReader(csvfile) for row in reader: - key = row.pop('name') - commodities[key] = row + commodities[int(row['id'])] = row # index by int for easier lookup and sorting size_pre = len(commodities) for commodity in data['lastStarport'].get('commodities'): - key = commodity['name'] + key = int(commodity['id']) new = { 'id' : commodity['id'], + 'symbol' : commodity['name'], 'category' : commodity['categoryname'], + 'name' : commodity['locName'], } old = commodities.get(key) if old: - if new['id'] != old['id'] or new['category'] != old['category']: + if new['symbol'] != old['symbol'] or new['name'] != old['name']: raise AssertionError('%s: "%s"!="%s"' % (key, new, old)) commodities[key] = new @@ -51,12 +52,10 @@ def addcommodities(data): os.rename(commodityfile, commodityfile+'.bak') with open(commodityfile, 'wb') as csvfile: - writer = csv.DictWriter(csvfile, ['id','category', 'name']) + writer = csv.DictWriter(csvfile, ['id', 'symbol', 'category', 'name']) writer.writeheader() - for key in commodities: - commodities[key]['name'] = key - for row in sorted(commodities.values(), key = lambda x: (x['category'], x['name'])): - writer.writerow(row) + for key in sorted(commodities): + writer.writerow(commodities[key]) print 'Added %d new commodities' % (len(commodities) - size_pre) @@ -172,8 +171,7 @@ if __name__ == "__main__": print 'No starport!' else: if data['lastStarport'].get('commodities'): - fixed = companion.fixup(data) - addcommodities(fixed) + addcommodities(data) else: print 'No market' if data['lastStarport'].get('modules'): diff --git a/commodity.csv b/commodity.csv new file mode 100644 index 00000000..a4656267 --- /dev/null +++ b/commodity.csv @@ -0,0 +1,203 @@ +id,symbol,category,name +128049152,Platinum,Metals,Platinum +128049153,Palladium,Metals,Palladium +128049154,Gold,Metals,Gold +128049155,Silver,Metals,Silver +128049156,Bertrandite,Minerals,Bertrandite +128049157,Indite,Minerals,Indite +128049158,Gallite,Minerals,Gallite +128049159,Coltan,Minerals,Coltan +128049160,Uraninite,Minerals,Uraninite +128049161,Lepidolite,Minerals,Lepidolite +128049162,Cobalt,Metals,Cobalt +128049163,Rutile,Minerals,Rutile +128049165,Bauxite,Minerals,Bauxite +128049166,Water,Chemicals,Water +128049168,Beryllium,Metals,Beryllium +128049169,Indium,Metals,Indium +128049170,Gallium,Metals,Gallium +128049171,Tantalum,Metals,Tantalum +128049172,Uranium,Metals,Uranium +128049173,Lithium,Metals,Lithium +128049174,Titanium,Metals,Titanium +128049175,Copper,Metals,Copper +128049176,Aluminium,Metals,Aluminium +128049177,Algae,Foods,Algae +128049178,FruitAndVegetables,Foods,Fruit and Vegetables +128049180,Grain,Foods,Grain +128049182,Animalmeat,Foods,Animal Meat +128049183,Fish,Foods,Fish +128049184,FoodCartridges,Foods,Food Cartridges +128049185,SyntheticMeat,Foods,Synthetic Meat +128049188,Tea,Foods,Tea +128049189,Coffee,Foods,Coffee +128049190,Leather,Textiles,Leather +128049191,NaturalFabrics,Textiles,Natural Fabrics +128049193,SyntheticFabrics,Textiles,Synthetic Fabrics +128049197,Polymers,Industrial Materials,Polymers +128049199,Semiconductors,Industrial Materials,Semiconductors +128049200,Superconductors,Industrial Materials,Superconductors +128049202,HydrogenFuel,Chemicals,Hydrogen Fuel +128049203,MineralOil,Chemicals,Mineral Oil +128049204,Explosives,Chemicals,Explosives +128049205,Pesticides,Chemicals,Pesticides +128049208,AgriculturalMedicines,Medicines,Agri-Medicines +128049209,PerformanceEnhancers,Medicines,Performance Enhancers +128049210,BasicMedicines,Medicines,Basic Medicines +128049212,BasicNarcotics,Legal Drugs,Narcotics +128049213,Tobacco,Legal Drugs,Tobacco +128049214,Beer,Legal Drugs,Beer +128049215,Wine,Legal Drugs,Wine +128049216,Liquor,Legal Drugs,Liquor +128049217,PowerGenerators,Machinery,Power Generators +128049218,WaterPurifiers,Machinery,Water Purifiers +128049220,HeliostaticFurnaces,Machinery,Microbial Furnaces +128049221,MineralExtractors,Machinery,Mineral Extractors +128049222,CropHarvesters,Machinery,Crop Harvesters +128049223,MarineSupplies,Machinery,Marine Equipment +128049225,ComputerComponents,Technology,Computer Components +128049226,HazardousEnvironmentSuits,Technology,H.E. Suits +128049227,Robotics,Technology,Robotics +128049228,AutoFabricators,Technology,Auto-Fabricators +128049229,AnimalMonitors,Technology,Animal Monitors +128049230,AquaponicSystems,Technology,Aquaponic Systems +128049231,AdvancedCatalysers,Technology,Advanced Catalysers +128049232,TerrainEnrichmentSystems,Technology,Land Enrichment Systems +128049233,PersonalWeapons,Weapons,Personal Weapons +128049234,BattleWeapons,Weapons,Battle Weapons +128049235,ReactiveArmour,Weapons,Reactive Armour +128049236,NonLethalWeapons,Weapons,Non-Lethal Weapons +128049238,DomesticAppliances,Consumer Items,Domestic Appliances +128049240,ConsumerTechnology,Consumer Items,Consumer Technology +128049241,Clothing,Consumer Items,Clothing +128049243,Slaves,Slavery,Slaves +128049244,Biowaste,Waste,Biowaste +128049245,ToxicWaste,Waste,Toxic Waste +128049246,ChemicalWaste,Waste,Chemical Waste +128049248,Scrap,Waste,Scrap +128049669,ProgenitorCells,Medicines,Progenitor Cells +128049670,CombatStabilisers,Medicines,Combat Stabilisers +128049671,ResonatingSeparators,Technology,Resonating Separators +128049672,BioReducingLichen,Technology,Bioreducing Lichen +128064028,AtmosphericExtractors,Machinery,Atmospheric Processors +128066403,Drones,NonMarketable,Limpets +128666752,USSCargoBlackBox,Salvage,Black Box +128666754,USSCargoTradeData,Salvage,Trade Data +128666755,USSCargoMilitaryPlans,Salvage,Military Plans +128666756,USSCargoAncientArtefact,Salvage,Ancient Artefact +128666757,USSCargoRareArtwork,Salvage,Rare Artwork +128666758,USSCargoExperimentalChemicals,Salvage,Experimental Chemicals +128666759,USSCargoRebelTransmissions,Salvage,Rebel Transmissions +128666760,USSCargoPrototypeTech,Salvage,Prototype Tech +128666761,USSCargoTechnicalBlueprints,Salvage,Technical Blueprints +128667728,ImperialSlaves,Slavery,Imperial Slaves +128668547,UnknownArtifact,Salvage,Unknown Artefact +128668548,AiRelics,Salvage,AI Relics +128668549,Hafnium178,Metals,Hafnium 178 +128668550,Painite,Minerals,Painite +128668551,Antiquities,Salvage,Antiquities +128668552,MilitaryIntelligence,Salvage,Military Intelligence +128671118,Osmium,Metals,Osmium +128671443,SAP8CoreContainer,Salvage,SAP 8 Core Container +128671444,TrinketsOfFortune,Consumer Items,Trinkets of Hidden Fortune +128672123,WreckageComponents,Salvage,Wreckage Components +128672124,EncriptedDataStorage,Salvage,Encrypted Data Storage +128672125,OccupiedCryoPod,Salvage,Occupied Escape Pod +128672126,PersonalEffects,Salvage,Personal Effects +128672127,ComercialSamples,Salvage,Commercial Samples +128672128,TacticalData,Salvage,Tactical Data +128672129,AssaultPlans,Salvage,Assault Plans +128672130,EncryptedCorrespondence,Salvage,Encrypted Correspondence +128672131,DiplomaticBag,Salvage,Diplomatic Bag +128672132,ScientificResearch,Salvage,Scientific Research +128672133,ScientificSamples,Salvage,Scientific Samples +128672134,PoliticalPrisoner,Salvage,Political Prisoner +128672135,Hostage,Salvage,Hostage +128672136,LargeExplorationDataCash,Salvage,Large Survey Data Cache +128672137,SmallExplorationDataCash,Salvage,Small Survey Data Cache +128672159,AntiqueJewellery,Salvage,Antique Jewellery +128672160,PreciousGems,Salvage,Precious Gems +128672161,EarthRelics,Salvage,Earth Relics +128672162,GeneBank,Salvage,Gene Bank +128672163,TimeCapsule,Salvage,Time Capsule +128672294,Cryolite,Minerals,Cryolite +128672295,Goslarite,Minerals,Goslarite +128672296,Moissanite,Minerals,Moissanite +128672297,Pyrophyllite,Minerals,Pyrophyllite +128672298,Lanthanum,Metals,Lanthanum +128672299,Thallium,Metals,Thallium +128672300,Bismuth,Metals,Bismuth +128672301,Thorium,Metals,Thorium +128672302,CeramicComposites,Industrial Materials,Ceramic Composites +128672303,SyntheticReagents,Chemicals,Synthetic Reagents +128672304,NerveAgents,Chemicals,Nerve Agents +128672305,SurfaceStabilisers,Chemicals,Surface Stabilisers +128672306,BootlegLiquor,Legal Drugs,Bootleg Liquor +128672307,GeologicalEquipment,Machinery,Geological Equipment +128672308,ThermalCoolingUnits,Machinery,Thermal Cooling Units +128672309,BuildingFabricators,Machinery,Building Fabricators +128672310,MuTomImager,Technology,Muon Imager +128672311,StructuralRegulators,Technology,Structural Regulators +128672312,Landmines,Weapons,Landmines +128672313,SkimerComponents,Machinery,Skimmer Components +128672314,EvacuationShelter,Consumer Items,Evacuation Shelter +128672315,GeologicalSamples,Salvage,Geological Samples +128672701,MetaAlloys,Industrial Materials,Meta-Alloys +128672775,Taaffeite,Minerals,Taaffeite +128672776,Jadeite,Minerals,Jadeite +128672810,UnstableDataCore,Salvage,Unstable Data Core +128672811,DamagedEscapePod,Salvage,Damaged Escape Pod +128673845,Praseodymium,Metals,Praseodymium +128673846,Bromellite,Minerals,Bromellite +128673847,Samarium,Metals,Samarium +128673848,LowTemperatureDiamond,Minerals,Low Temperature Diamonds +128673850,HydrogenPeroxide,Chemicals,Hydrogen Peroxide +128673851,LiquidOxygen,Chemicals,Liquid oxygen +128673852,MethanolMonohydrateCrystals,Minerals,Methanol Monohydrate Crystals +128673853,LithiumHydroxide,Minerals,Lithium Hydroxide +128673854,MethaneClathrate,Minerals,Methane Clathrate +128673855,InsulatingMembrane,Industrial Materials,Insulating Membrane +128673856,CMMComposite,Industrial Materials,CMM Composite +128673857,CoolingHoses,Industrial Materials,Micro-weave Cooling Hoses +128673858,NeofabricInsulation,Industrial Materials,Neofabric Insulation +128673859,ArticulationMotors,Machinery,Articulation Motors +128673860,HNShockMount,Machinery,HN Shock Mount +128673861,EmergencyPowerCells,Machinery,Emergency Power Cells +128673862,PowerConverter,Machinery,Power Converter +128673863,PowerGridAssembly,Machinery,Energy Grid Assembly +128673864,PowerTransferConduits,Machinery,Power Transfer Bus +128673865,RadiationBaffle,Machinery,Radiation Baffle +128673866,ExhaustManifold,Machinery,Exhaust Manifold +128673867,ReinforcedMountingPlate,Machinery,Reinforced Mounting Plate +128673868,HeatsinkInterlink,Machinery,Heatsink Interlink +128673869,MagneticEmitterCoil,Machinery,Magnetic Emitter Coil +128673870,ModularTerminals,Machinery,Modular Terminals +128673871,Nanobreakers,Technology,Nanobreakers +128673872,TelemetrySuite,Technology,Telemetry Suite +128673873,MicroControllers,Technology,Micro Controllers +128673874,IonDistributor,Machinery,Ion Distributor +128673875,DiagnosticSensor,Technology,Hardware Diagnostic Sensor +128673876,UnknownArtifact2,Salvage,Unknown Probe +128682044,ConductiveFabrics,Textiles,Conductive Fabrics +128682045,MilitaryGradeFabrics,Textiles,Military Grade Fabrics +128682046,AdvancedMedicines,Medicines,Advanced Medicines +128682047,MedicalDiagnosticEquipment,Technology,Medical Diagnostic Equipment +128682048,SurvivalEquipment,Consumer Items,Survival Equipment +128682049,DataCore,Salvage,Data Core +128682050,GalacticTravelGuide,Salvage,Galactic Travel Guide +128682051,MysteriousIdol,Salvage,Mysterious Idol +128682052,ProhibitedResearchMaterials,Salvage,Prohibited Research Materials +128682053,AntimatterContainmentUnit,Salvage,Antimatter Containment Unit +128682054,SpacePioneerRelics,Salvage,Space Pioneer Relics +128682055,FossilRemnants,Salvage,Fossil Remnants +128732183,AncientRelic,Salvage,Ancient Relic +128732184,AncientOrb,Salvage,Ancient Orb +128732185,AncientCasket,Salvage,Ancient Casket +128732186,AncientTablet,Salvage,Ancient Tablet +128732187,AncientUrn,Salvage,Ancient Urn +128732188,AncientTotem,Salvage,Ancient Totem +128737287,UnknownResin,Salvage,Thargoid Resin +128737288,UnknownBiologicalMatter,Salvage,Thargoid Biological Matter +128737289,UnknownTechnologySamples,Salvage,Thargoid Technology Samples +128740752,UnknownArtifact3,Salvage,Thargoid Link +128793114,PlatinumAloy,Metals,Platinum Aloy diff --git a/companion.py b/companion.py index 8738902e..61e89e53 100644 --- a/companion.py +++ b/companion.py @@ -1,3 +1,4 @@ +import csv import requests from cookielib import LWPCookieJar import hashlib @@ -34,6 +35,8 @@ category_map = { 'NonMarketable' : False, # Don't appear in the in-game market so don't report } +commodity_map = {} + ship_map = { 'adder' : 'Adder', 'anaconda' : 'Anaconda', @@ -277,8 +280,16 @@ class Session: print ('Content:\n%s' % r.text).encode('utf-8') -# Returns a shallow copy of the received data with anomalies in the commodity data fixed up +# Returns a shallow copy of the received data suitable for export to older tools - English commodity names and anomalies fixed up def fixup(data): + + if not commodity_map: + # Lazily populate + with open(join(config.respath, 'commodity.csv'), 'rb') as csvfile: + reader = csv.DictReader(csvfile) + for row in reader: + commodity_map[row['symbol']] = (row['category'], row['name']) + commodities = [] for commodity in data['lastStarport'].get('commodities') or []: @@ -305,7 +316,10 @@ def fixup(data): else: # Rewrite text fields new = dict(commodity) # shallow copy - new['categoryname'] = category_map.get(commodity['categoryname'], commodity['categoryname']) + if commodity['name'] in commodity_map: + (new['categoryname'], new['name']) = commodity_map[commodity['name']] + elif commodity['categoryname'] in category_map: + new['categoryname'] = category_map[commodity['categoryname']] # Force demand and stock to zero if their corresponding bracket is zero # Fixes spurious "demand": 1 in ED 1.3 diff --git a/setup.py b/setup.py index 5897fa61..dd050865 100755 --- a/setup.py +++ b/setup.py @@ -76,7 +76,7 @@ if sys.platform=='darwin': 'excludes': [ 'certifi', 'distutils', 'iniparse', '_markerlib', 'PIL', 'pkg_resources', 'simplejson', 'unittest' ], 'iconfile': '%s.icns' % APPNAME, 'include_plugins': [('plugins', x) for x in PLUGINS], - 'resources': ['snd_good.wav', 'snd_bad.wav', 'modules.p', 'ships.p', 'stations.p', 'systems.p'], + 'resources': ['commodity.csv', 'snd_good.wav', 'snd_bad.wav', 'modules.p', 'ships.p', 'stations.p', 'systems.p'], 'semi_standalone': True, 'site_packages': False, 'plist': { @@ -116,6 +116,7 @@ elif sys.platform=='win32': requests.certs.where(), 'WinSparkle.dll', 'WinSparkle.pdb', # For debugging - don't include in package + 'commodity.csv', 'snd_good.wav', 'snd_bad.wav', 'modules.p',