Collate commodities indexed by id and including symbolic name

Use collated info in export to older tools.

EDMC.py

@@ -177,17 +177,16 @@ try:
 
     # Finally - the data looks sane and we're docked at a station
 
-    # Fixup anomalies in the commodity data
-    fixed = companion.fixup(data)
-
     if args.j:
         # Collate from JSON dump
-        collate.addcommodities(fixed)
+        collate.addcommodities(data)
-        collate.addmodules(fixed)
+        collate.addmodules(data)
-        collate.addships(fixed)
+        collate.addships(data)
 
     if args.m:
         if data['lastStarport'].get('commodities'):
+            # Fixup anomalies in the commodity data
+            fixed = companion.fixup(data)
             commodity.export(fixed, COMMODITY_DEFAULT, args.m)
         else:
             sys.stderr.write("Station doesn't have a market\n")

EDMarketConnector.wxs

@@ -123,6 +123,9 @@
 					<Component Guid="{62DF41B7-0BE8-48F3-BB22-90E9201A6D8C}">
 						<File KeyPath="yes" Source="SourceDir\cacert.pem" />
 					</Component>
+					<Component Guid="*">
+						<File KeyPath="yes" Source="SourceDir\commodity.csv" />
+					</Component>
 					<Component Guid="{6762E871-5FA1-4C2F-A3C9-6A9954CC018C}">
 						<File KeyPath="yes" Source="SourceDir\EDMarketConnector.ico" />
 					</Component>
@@ -455,6 +458,7 @@
 			<ComponentRef Id="_tkinter.pyd" />
 			<ComponentRef Id="bz2.pyd" />
 			<ComponentRef Id="cacert.pem" />
+			<ComponentRef Id="commodity.csv" />
 			<ComponentRef Id="cs.strings" />
 			<ComponentRef Id="de.strings" />
 			<ComponentRef Id="EDMarketConnector.ico" />

collate.py

@@ -27,19 +27,20 @@ def addcommodities(data):
         with open(commodityfile) as csvfile:
             reader = csv.DictReader(csvfile)
             for row in reader:
-                key = row.pop('name')
+                commodities[int(row['id'])] = row	# index by int for easier lookup and sorting
-                commodities[key] = row
     size_pre = len(commodities)
 
     for commodity in data['lastStarport'].get('commodities'):
-        key = commodity['name']
+        key = int(commodity['id'])
         new = {
             'id'       : commodity['id'],
+            'symbol'   : commodity['name'],
             'category' : commodity['categoryname'],
+            'name'     : commodity['locName'],
         }
         old = commodities.get(key)
         if old:
-            if new['id'] != old['id'] or new['category'] != old['category']:
+            if new['symbol'] != old['symbol'] or new['name'] != old['name']:
                 raise AssertionError('%s: "%s"!="%s"' % (key, new, old))
         commodities[key] = new
 
@@ -51,12 +52,10 @@ def addcommodities(data):
             os.rename(commodityfile, commodityfile+'.bak')
 
         with open(commodityfile, 'wb') as csvfile:
-            writer = csv.DictWriter(csvfile, ['id','category', 'name'])
+            writer = csv.DictWriter(csvfile, ['id', 'symbol', 'category', 'name'])
             writer.writeheader()
-            for key in commodities:
+            for key in sorted(commodities):
-                commodities[key]['name'] = key
+                writer.writerow(commodities[key])
-            for row in sorted(commodities.values(), key = lambda x: (x['category'], x['name'])):
-                writer.writerow(row)
 
         print 'Added %d new commodities' % (len(commodities) - size_pre)
 
@@ -172,8 +171,7 @@ if __name__ == "__main__":
                     print 'No starport!'
                 else:
                     if data['lastStarport'].get('commodities'):
-                        fixed = companion.fixup(data)
+                        addcommodities(data)
-                        addcommodities(fixed)
                     else:
                         print 'No market'
                     if data['lastStarport'].get('modules'):

commodity.csv

@@ -0,0 +1,203 @@
+id,symbol,category,name
+128049152,Platinum,Metals,Platinum
+128049153,Palladium,Metals,Palladium
+128049154,Gold,Metals,Gold
+128049155,Silver,Metals,Silver
+128049156,Bertrandite,Minerals,Bertrandite
+128049157,Indite,Minerals,Indite
+128049158,Gallite,Minerals,Gallite
+128049159,Coltan,Minerals,Coltan
+128049160,Uraninite,Minerals,Uraninite
+128049161,Lepidolite,Minerals,Lepidolite
+128049162,Cobalt,Metals,Cobalt
+128049163,Rutile,Minerals,Rutile
+128049165,Bauxite,Minerals,Bauxite
+128049166,Water,Chemicals,Water
+128049168,Beryllium,Metals,Beryllium
+128049169,Indium,Metals,Indium
+128049170,Gallium,Metals,Gallium
+128049171,Tantalum,Metals,Tantalum
+128049172,Uranium,Metals,Uranium
+128049173,Lithium,Metals,Lithium
+128049174,Titanium,Metals,Titanium
+128049175,Copper,Metals,Copper
+128049176,Aluminium,Metals,Aluminium
+128049177,Algae,Foods,Algae
+128049178,FruitAndVegetables,Foods,Fruit and Vegetables
+128049180,Grain,Foods,Grain
+128049182,Animalmeat,Foods,Animal Meat
+128049183,Fish,Foods,Fish
+128049184,FoodCartridges,Foods,Food Cartridges
+128049185,SyntheticMeat,Foods,Synthetic Meat
+128049188,Tea,Foods,Tea
+128049189,Coffee,Foods,Coffee
+128049190,Leather,Textiles,Leather
+128049191,NaturalFabrics,Textiles,Natural Fabrics
+128049193,SyntheticFabrics,Textiles,Synthetic Fabrics
+128049197,Polymers,Industrial Materials,Polymers
+128049199,Semiconductors,Industrial Materials,Semiconductors
+128049200,Superconductors,Industrial Materials,Superconductors
+128049202,HydrogenFuel,Chemicals,Hydrogen Fuel
+128049203,MineralOil,Chemicals,Mineral Oil
+128049204,Explosives,Chemicals,Explosives
+128049205,Pesticides,Chemicals,Pesticides
+128049208,AgriculturalMedicines,Medicines,Agri-Medicines
+128049209,PerformanceEnhancers,Medicines,Performance Enhancers
+128049210,BasicMedicines,Medicines,Basic Medicines
+128049212,BasicNarcotics,Legal Drugs,Narcotics
+128049213,Tobacco,Legal Drugs,Tobacco
+128049214,Beer,Legal Drugs,Beer
+128049215,Wine,Legal Drugs,Wine
+128049216,Liquor,Legal Drugs,Liquor
+128049217,PowerGenerators,Machinery,Power Generators
+128049218,WaterPurifiers,Machinery,Water Purifiers
+128049220,HeliostaticFurnaces,Machinery,Microbial Furnaces
+128049221,MineralExtractors,Machinery,Mineral Extractors
+128049222,CropHarvesters,Machinery,Crop Harvesters
+128049223,MarineSupplies,Machinery,Marine Equipment
+128049225,ComputerComponents,Technology,Computer Components
+128049226,HazardousEnvironmentSuits,Technology,H.E. Suits
+128049227,Robotics,Technology,Robotics
+128049228,AutoFabricators,Technology,Auto-Fabricators
+128049229,AnimalMonitors,Technology,Animal Monitors
+128049230,AquaponicSystems,Technology,Aquaponic Systems
+128049231,AdvancedCatalysers,Technology,Advanced Catalysers
+128049232,TerrainEnrichmentSystems,Technology,Land Enrichment Systems
+128049233,PersonalWeapons,Weapons,Personal Weapons
+128049234,BattleWeapons,Weapons,Battle Weapons
+128049235,ReactiveArmour,Weapons,Reactive Armour
+128049236,NonLethalWeapons,Weapons,Non-Lethal Weapons
+128049238,DomesticAppliances,Consumer Items,Domestic Appliances
+128049240,ConsumerTechnology,Consumer Items,Consumer Technology
+128049241,Clothing,Consumer Items,Clothing
+128049243,Slaves,Slavery,Slaves
+128049244,Biowaste,Waste,Biowaste
+128049245,ToxicWaste,Waste,Toxic Waste
+128049246,ChemicalWaste,Waste,Chemical Waste
+128049248,Scrap,Waste,Scrap
+128049669,ProgenitorCells,Medicines,Progenitor Cells
+128049670,CombatStabilisers,Medicines,Combat Stabilisers
+128049671,ResonatingSeparators,Technology,Resonating Separators
+128049672,BioReducingLichen,Technology,Bioreducing Lichen
+128064028,AtmosphericExtractors,Machinery,Atmospheric Processors
+128066403,Drones,NonMarketable,Limpets
+128666752,USSCargoBlackBox,Salvage,Black Box
+128666754,USSCargoTradeData,Salvage,Trade Data
+128666755,USSCargoMilitaryPlans,Salvage,Military Plans
+128666756,USSCargoAncientArtefact,Salvage,Ancient Artefact
+128666757,USSCargoRareArtwork,Salvage,Rare Artwork
+128666758,USSCargoExperimentalChemicals,Salvage,Experimental Chemicals
+128666759,USSCargoRebelTransmissions,Salvage,Rebel Transmissions
+128666760,USSCargoPrototypeTech,Salvage,Prototype Tech
+128666761,USSCargoTechnicalBlueprints,Salvage,Technical Blueprints
+128667728,ImperialSlaves,Slavery,Imperial Slaves
+128668547,UnknownArtifact,Salvage,Unknown Artefact
+128668548,AiRelics,Salvage,AI Relics
+128668549,Hafnium178,Metals,Hafnium 178
+128668550,Painite,Minerals,Painite
+128668551,Antiquities,Salvage,Antiquities
+128668552,MilitaryIntelligence,Salvage,Military Intelligence
+128671118,Osmium,Metals,Osmium
+128671443,SAP8CoreContainer,Salvage,SAP 8 Core Container
+128671444,TrinketsOfFortune,Consumer Items,Trinkets of Hidden Fortune
+128672123,WreckageComponents,Salvage,Wreckage Components
+128672124,EncriptedDataStorage,Salvage,Encrypted Data Storage
+128672125,OccupiedCryoPod,Salvage,Occupied Escape Pod
+128672126,PersonalEffects,Salvage,Personal Effects
+128672127,ComercialSamples,Salvage,Commercial Samples
+128672128,TacticalData,Salvage,Tactical Data
+128672129,AssaultPlans,Salvage,Assault Plans
+128672130,EncryptedCorrespondence,Salvage,Encrypted Correspondence
+128672131,DiplomaticBag,Salvage,Diplomatic Bag
+128672132,ScientificResearch,Salvage,Scientific Research
+128672133,ScientificSamples,Salvage,Scientific Samples
+128672134,PoliticalPrisoner,Salvage,Political Prisoner
+128672135,Hostage,Salvage,Hostage
+128672136,LargeExplorationDataCash,Salvage,Large Survey Data Cache
+128672137,SmallExplorationDataCash,Salvage,Small Survey Data Cache
+128672159,AntiqueJewellery,Salvage,Antique Jewellery 
+128672160,PreciousGems,Salvage,Precious Gems
+128672161,EarthRelics,Salvage,Earth Relics 
+128672162,GeneBank,Salvage,Gene Bank
+128672163,TimeCapsule,Salvage,Time Capsule
+128672294,Cryolite,Minerals,Cryolite
+128672295,Goslarite,Minerals,Goslarite
+128672296,Moissanite,Minerals,Moissanite
+128672297,Pyrophyllite,Minerals,Pyrophyllite
+128672298,Lanthanum,Metals,Lanthanum
+128672299,Thallium,Metals,Thallium
+128672300,Bismuth,Metals,Bismuth
+128672301,Thorium,Metals,Thorium
+128672302,CeramicComposites,Industrial Materials,Ceramic Composites
+128672303,SyntheticReagents,Chemicals,Synthetic Reagents
+128672304,NerveAgents,Chemicals,Nerve Agents
+128672305,SurfaceStabilisers,Chemicals,Surface Stabilisers
+128672306,BootlegLiquor,Legal Drugs,Bootleg Liquor
+128672307,GeologicalEquipment,Machinery,Geological Equipment
+128672308,ThermalCoolingUnits,Machinery,Thermal Cooling Units
+128672309,BuildingFabricators,Machinery,Building Fabricators
+128672310,MuTomImager,Technology,Muon Imager
+128672311,StructuralRegulators,Technology,Structural Regulators
+128672312,Landmines,Weapons,Landmines
+128672313,SkimerComponents,Machinery,Skimmer Components
+128672314,EvacuationShelter,Consumer Items,Evacuation Shelter
+128672315,GeologicalSamples,Salvage,Geological Samples
+128672701,MetaAlloys,Industrial Materials,Meta-Alloys
+128672775,Taaffeite,Minerals,Taaffeite
+128672776,Jadeite,Minerals,Jadeite
+128672810,UnstableDataCore,Salvage,Unstable Data Core
+128672811,DamagedEscapePod,Salvage,Damaged Escape Pod
+128673845,Praseodymium,Metals,Praseodymium
+128673846,Bromellite,Minerals,Bromellite
+128673847,Samarium,Metals,Samarium
+128673848,LowTemperatureDiamond,Minerals,Low Temperature Diamonds
+128673850,HydrogenPeroxide,Chemicals,Hydrogen Peroxide
+128673851,LiquidOxygen,Chemicals,Liquid oxygen
+128673852,MethanolMonohydrateCrystals,Minerals,Methanol Monohydrate Crystals
+128673853,LithiumHydroxide,Minerals,Lithium Hydroxide
+128673854,MethaneClathrate,Minerals,Methane Clathrate
+128673855,InsulatingMembrane,Industrial Materials,Insulating Membrane
+128673856,CMMComposite,Industrial Materials,CMM Composite
+128673857,CoolingHoses,Industrial Materials,Micro-weave Cooling Hoses
+128673858,NeofabricInsulation,Industrial Materials,Neofabric Insulation
+128673859,ArticulationMotors,Machinery,Articulation Motors
+128673860,HNShockMount,Machinery,HN Shock Mount
+128673861,EmergencyPowerCells,Machinery,Emergency Power Cells
+128673862,PowerConverter,Machinery,Power Converter
+128673863,PowerGridAssembly,Machinery,Energy Grid Assembly
+128673864,PowerTransferConduits,Machinery,Power Transfer Bus
+128673865,RadiationBaffle,Machinery,Radiation Baffle
+128673866,ExhaustManifold,Machinery,Exhaust Manifold
+128673867,ReinforcedMountingPlate,Machinery,Reinforced Mounting Plate
+128673868,HeatsinkInterlink,Machinery,Heatsink Interlink
+128673869,MagneticEmitterCoil,Machinery,Magnetic Emitter Coil
+128673870,ModularTerminals,Machinery,Modular Terminals
+128673871,Nanobreakers,Technology,Nanobreakers
+128673872,TelemetrySuite,Technology,Telemetry Suite
+128673873,MicroControllers,Technology,Micro Controllers
+128673874,IonDistributor,Machinery,Ion Distributor
+128673875,DiagnosticSensor,Technology,Hardware Diagnostic Sensor
+128673876,UnknownArtifact2,Salvage,Unknown Probe
+128682044,ConductiveFabrics,Textiles,Conductive Fabrics
+128682045,MilitaryGradeFabrics,Textiles,Military Grade Fabrics
+128682046,AdvancedMedicines,Medicines,Advanced Medicines
+128682047,MedicalDiagnosticEquipment,Technology,Medical Diagnostic Equipment
+128682048,SurvivalEquipment,Consumer Items,Survival Equipment
+128682049,DataCore,Salvage,Data Core
+128682050,GalacticTravelGuide,Salvage,Galactic Travel Guide
+128682051,MysteriousIdol,Salvage,Mysterious Idol
+128682052,ProhibitedResearchMaterials,Salvage,Prohibited Research Materials
+128682053,AntimatterContainmentUnit,Salvage,Antimatter Containment Unit
+128682054,SpacePioneerRelics,Salvage,Space Pioneer Relics
+128682055,FossilRemnants,Salvage,Fossil Remnants
+128732183,AncientRelic,Salvage,Ancient Relic
+128732184,AncientOrb,Salvage,Ancient Orb
+128732185,AncientCasket,Salvage,Ancient Casket
+128732186,AncientTablet,Salvage,Ancient Tablet
+128732187,AncientUrn,Salvage,Ancient Urn
+128732188,AncientTotem,Salvage,Ancient Totem
+128737287,UnknownResin,Salvage,Thargoid Resin
+128737288,UnknownBiologicalMatter,Salvage,Thargoid Biological Matter
+128737289,UnknownTechnologySamples,Salvage,Thargoid Technology Samples
+128740752,UnknownArtifact3,Salvage,Thargoid Link
+128793114,PlatinumAloy,Metals,Platinum Aloy

companion.py

@@ -1,3 +1,4 @@
+import csv
 import requests
 from cookielib import LWPCookieJar
 import hashlib
@@ -34,6 +35,8 @@ category_map = {
     'NonMarketable' : False,	# Don't appear in the in-game market so don't report
 }
 
+commodity_map = {}
+
 ship_map = {
     'adder'                       : 'Adder',
     'anaconda'                    : 'Anaconda',
@@ -277,8 +280,16 @@ class Session:
             print ('Content:\n%s' % r.text).encode('utf-8')
 
 
-# Returns a shallow copy of the received data with anomalies in the commodity data fixed up
+# Returns a shallow copy of the received data suitable for export to older tools - English commodity names and anomalies fixed up
 def fixup(data):
+
+    if not commodity_map:
+        # Lazily populate
+        with open(join(config.respath, 'commodity.csv'), 'rb') as csvfile:
+            reader = csv.DictReader(csvfile)
+            for row in reader:
+                commodity_map[row['symbol']] = (row['category'], row['name'])
+
     commodities = []
     for commodity in data['lastStarport'].get('commodities') or []:
 
@@ -305,7 +316,10 @@ def fixup(data):
             else:
                 # Rewrite text fields
                 new = dict(commodity)	# shallow copy
-                new['categoryname'] = category_map.get(commodity['categoryname'], commodity['categoryname'])
+                if commodity['name'] in commodity_map:
+                    (new['categoryname'], new['name']) = commodity_map[commodity['name']]
+                elif commodity['categoryname'] in category_map:
+                    new['categoryname'] = category_map[commodity['categoryname']]
 
                 # Force demand and stock to zero if their corresponding bracket is zero
                 # Fixes spurious "demand": 1 in ED 1.3

setup.py

@@ -76,7 +76,7 @@ if sys.platform=='darwin':
                   'excludes': [ 'certifi', 'distutils', 'iniparse', '_markerlib', 'PIL', 'pkg_resources', 'simplejson', 'unittest' ],
                   'iconfile': '%s.icns' % APPNAME,
                   'include_plugins': [('plugins', x) for x in PLUGINS],
-                  'resources': ['snd_good.wav', 'snd_bad.wav', 'modules.p', 'ships.p', 'stations.p', 'systems.p'],
+                  'resources': ['commodity.csv', 'snd_good.wav', 'snd_bad.wav', 'modules.p', 'ships.p', 'stations.p', 'systems.p'],
                   'semi_standalone': True,
                   'site_packages': False,
                   'plist': {
@@ -116,6 +116,7 @@ elif sys.platform=='win32':
             requests.certs.where(),
             'WinSparkle.dll',
             'WinSparkle.pdb',	# For debugging - don't include in package
+            'commodity.csv',
             'snd_good.wav',
             'snd_bad.wav',
             'modules.p',