conda create -n citylearn python=3.10 -y
conda activate citylearn

git clone https://github.com/intelligent-environments-lab/CityLearn.git
cd CityLearn
pip install -e .    # 源码控制版，仅用于复现
pip install citylearn   # 后续试验可以直接用这个下载

ERROR: Ignored the following versions that require a different python version: 2.3.0 Requires-Python >=3.11; 2.3.1 Requires-Python >=3.11; 2.3.2 Requires-Python >=3.11; 2.3.3 Requires-Python >=3.11; 2.3.4 Requires-Python >=3.11
ERROR: Could not find a version that satisfies the requirement openstudio<=3.3.0 (from citylearn) (from versions: 3.5.0, 3.5.1, 3.6.0, 3.6.1, 3.7.0rc1, 3.7.0, 3.8.0, 3.9.0, 3.10.0rc5, 3.10.0)
ERROR: No matching distribution found for openstudio<=3.3.0

pip install -e . --no-deps
pip install numpy==1.26.4 pandas pyyaml scikit-learn==1.2.2 simplejson torch torchvision gymnasium==0.28.1 nrel-pysam doe_xstock

from citylearn.citylearn import CityLearnEnv
from pathlib import Path
import numpy as np

# 加载 CityLearn 环境
schema = Path('/root/CityLearn/data/datasets/citylearn_challenge_2021/schema.json')
env = CityLearnEnv(schema=schema)
print("✅ Env loaded successfully!")

# 重置环境（Gymnasium 风格）
obs, info = env.reset()

# 打印建筑数量与ID
print("✅ Number of buildings:", len(env.buildings))

# 打印每栋建筑的观测长度
obs_lens = [len(o) for o in obs]
print("✅ Observation lengths per building:", obs_lens)

# 示例：查看第一个建筑的前5个特征
print("✅ Sample observation (Building 1):", obs[0][:5])

# 如果要转换为矩阵，可以先对齐长度（取最短长度或补零）
min_len = min(obs_lens)
obs_array = np.array([o[:min_len] for o in obs])
print("✅ Converted obs array shape:", obs_array.shape)

(citylearnv2) root@autodl-container-491e4b9a34-9e5b11d5:~/CityLearn# python Untitled1.py
Couldn't import dot_parser, loading of dot files will not be possible.
✅ Env loaded successfully!
✅ Number of buildings: 9
✅ Observation lengths per building: [28, 27, 26, 27, 28, 28, 27, 27, 27]
✅ Sample observation (Building 1): [1, 4, 1, 9.4, 10.020000457763672]
✅ Converted obs array shape: (9, 26)

from citylearn.citylearn import CityLearnEnv
from citylearn.agents.base import BaselineAgent
from citylearn.agents.rbc import BasicRBC
from citylearn.agents.sac import SAC
from pathlib import Path

def run(agent_class, schema, episodes=1, central_agent=True, save_dir=None, **kwargs):
    print(f"\n=== Running {agent_class.__name__} ===")
    env = CityLearnEnv(schema, central_agent=central_agent)
    agent = agent_class(env)
    agent.learn(episodes=episodes, deterministic_finish=True)
    results = env.evaluate()
    env.close()
    print(f"✅ {agent_class.__name__} finished.")
    if save_dir:
        Path(save_dir).mkdir(parents=True, exist_ok=True)
        (Path(save_dir)/"metrics.txt").write_text(str(results))
    return results

def main():
    # 使用本地数据集 schema
    schema = "data/datasets/citylearn_challenge_2023_phase_2_local_evaluation/schema.json"

    # 1 Baseline 无控制
    res_base = run(BaselineAgent, schema, episodes=1, central_agent=True, save_dir="results/4_1_1_baseline")

    # 2 RBC 规则控制
    res_rbc = run(BasicRBC, schema, episodes=1, central_agent=True, save_dir="results/4_1_1_rbc")

    # 3 SAC 强化学习控制
    res_sac = run(SAC, schema, episodes=5, central_agent=False, save_dir="results/4_1_1_sac")

    # 输出对比结果
    print("\n=== Comparison ===")
    print("Baseline:", res_base)
    print("RBC:", res_rbc)
    print("SAC:", res_sac)

if __name__ == "__main__":
    main()

env = CityLearnEnv(schema, central_agent=central_agent)

agent = agent_class(env)

agent.learn(episodes=episodes, deterministic_finish=True)

results = env.evaluate()

from citylearn.agents.base import BaseAgent

class MyRBC(BaseAgent):
    def select_actions(self, observations):
        actions = []
        for obs in observations:
            price = obs['electricity_price']
            soc = obs['storage_soc']
            solar = obs['solar_generation']
            if price < 0.05 and solar > 0:
                action = 1.0      # 加强充电
            elif price > 0.2 or soc > 0.8:
                action = -1.0     # 放电
            else:
                action = 0.0
            actions.append(action)
        return actions
        
run(MyRBC, schema, episodes=1, central_agent=True)

from citylearn.agents.base import BaseAgent
from stable_baselines3 import PPO

class PPOAgent(BaseAgent):
    def __init__(self, env):
        super().__init__(env)
        self.model = PPO('MlpPolicy', env, verbose=1)

    def learn(self, episodes=1, **kwargs):
        self.model.learn(total_timesteps=episodes * len(self.env.time_steps))

    def select_actions(self, observations):
        action, _ = self.model.predict(observations, deterministic=True)
        return action

run(PPOAgent, schema, episodes=5, central_agent=False)

"reward_function": {
  "type": "citylearn.reward_function.RewardFunction",
  "attributes": {
    "electricity_cost_weight": 0.6,
    "emission_weight": 0.4,
    "peak_to_average_ratio_weight": 0.0
  }
}

"reward_function": {
  "type": "citylearn.reward_function.RewardFunction",
  "attributes": {
    "electricity_cost_weight": 0.0,
    "emission_weight": 0.0,
    "peak_to_average_ratio_weight": 1.0
  }
}

"reward_function": {
  "type": "citylearn.reward_function.ComfortReward",
  "attributes": {
    "band": 1.0,
    "lower_exponent": 2.0,
    "higher_exponent": 3.0,
    "m": 3.0,
    "mode": "cooling"
  }
}

# ============================================================
# CityLearn 4.2 实验复现: Energy storage system control
# 参考论文 Section 4.2
# ============================================================

from citylearn.citylearn import CityLearnEnv
from citylearn.agents.base import BaselineAgent
from citylearn.agents.rbc import BasicRBC
from citylearn.agents.sac import SAC
from pathlib import Path
from debug_sac import DebugSAC  # ✅ 使用调试版本，不改库



# 测试用，仅使用n栋buildings
def run(agent_class, schema, episodes=1, central_agent=False, save_dir=None):
    print(f"\n=== Running {agent_class.__name__} | Central Agent = {central_agent} ===")

    # ✅ 正确选择前三栋建筑
    env_full = CityLearnEnv(schema, central_agent=central_agent)
    selected_buildings = env_full.buildings[:20]
    env_full.close()

    # ✅ 用3栋建筑创建新环境
    env = CityLearnEnv(schema, buildings=selected_buildings, central_agent=central_agent)
    env.reset()

    agent = agent_class(env)
    agent.learn(episodes=episodes, deterministic_finish=True)

    results = env.evaluate()
    env.close()

    print(f"✅ {agent_class.__name__} finished.")
    print(f"📊 Results: {results}")

    if save_dir:
        Path(save_dir).mkdir(parents=True, exist_ok=True)
        (Path(save_dir) / "metrics.txt").write_text(str(results))

    return results


# def run(agent_class, schema, episodes=1, central_agent=False, save_dir=None):
#     print(f"\n=== Running {agent_class.__name__} | Central Agent = {central_agent} ===")

#     # ✅ 创建环境
#     env = CityLearnEnv(schema, buildings=env.buildings[:3], central_agent=False)
#     # env = CityLearnEnv(schema, central_agent=central_agent)
#     env.reset()

#     # ✅ 创建 Agent
#     agent = agent_class(env)

#     # ✅ 训练 Agent（Baseline / RBC 会自动跳过 learn）
#     agent.learn(episodes=episodes, deterministic_finish=True)

#     # ✅ 评估指标
#     results = env.evaluate()
#     env.close()

#     print(f"✅ {agent_class.__name__} finished.")
#     print(f"📊 Results: {results}")

#     # ✅ 保存结果
#     if save_dir:
#         Path(save_dir).mkdir(parents=True, exist_ok=True)
#         (Path(save_dir) / "metrics.txt").write_text(str(results))

#     return results

def main():
    schema = "data/datasets/ca_alameda_county_neighborhood/schema.json"
    central_agent = False  

    # Baseline
    run(
        agent_class=BaselineAgent,
        schema=schema,
        episodes=1,
        central_agent=central_agent,
        save_dir="results/4_2_baseline_CA"
    )

    # RBC
    run(
        agent_class=BasicRBC,
        schema=schema,
        episodes=1,
        central_agent=central_agent,
        save_dir="results/4_2_rbc_CA"
    )

    # SAC
    run(
        agent_class=DebugSAC,
        schema=schema,
        episodes=5,
        central_agent=central_agent,
        save_dir="results/4_2_sac_CA"
    )

if __name__ == "__main__":
    main()

AssertionError: demand is greater than heating_device max output | timestep: 0, building: resstock-amy2018-2021-release-1-122940, outage: False, demand: 3.1614, output: 2.9663, difference: 0.1950

# before
assert self.power_outage or demand <= max_device_output or abs(demand - max_device_output) < TOLERANCE, \
    f"demand is greater than heating_device max output | ..."

# now
if not (self.power_outage or demand <= max_device_output or abs(demand - max_device_output) < TOLERANCE):
    print(f"[Warning] demand > heating_device max output at timestep {self.time_step}")

TOLERANCE = 1e-4

(citylearnv2) root@autodl-container-491e4b9a34-9e5b11d5:~/CityLearn# python -O run_4_2.py
Couldn't import dot_parser, loading of dot files will not be possible.

=== Running SAC ===
Traceback (most recent call last):
  File "/root/CityLearn/run_4_2.py", line 44, in <module>
    main()
  File "/root/CityLearn/run_4_2.py", line 40, in main
    run(SAC, schema, episodes=5, central_agent=central_agent,
  File "/root/CityLearn/run_4_2.py", line 16, in run
    agent.learn(episodes=episodes, deterministic_finish=True)
  File "/root/CityLearn/citylearn/agents/base.py", line 156, in learn
    actions = self.predict(observations, deterministic=deterministic)
  File "/root/CityLearn/citylearn/agents/sac.py", line 189, in predict
    actions = self.get_post_exploration_prediction(observations, deterministic)
  File "/root/CityLearn/citylearn/agents/sac.py", line 206, in get_post_exploration_prediction
    o = torch.FloatTensor(o).unsqueeze(0).to(self.device)
TypeError: new(): data must be a sequence (got NoneType)

# inspect_schema.py
import json, sys, argparse, os
from typing import Any, Dict, List

def load_schema(path: str) -> Dict[str, Any]:
    txt = open(path, 'r', encoding='utf-8').read()
    # 先尝试 JSON
    try:
        data = json.loads(txt)
    except Exception:
        # 可选：YAML（若安装了 PyYAML）
        try:
            import yaml  # pip install pyyaml
            data = yaml.safe_load(txt)
        except Exception as e:
            print(f"❌ 既不是 JSON 也不是可解析的 YAML：{e}")
            sys.exit(1)
    # 大多数 CityLearn schema 顶层有个 'root'
    return data.get('root', data)

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('--schema', required=True, help='path/to/schema.json')
    args = ap.parse_args()

    sc = load_schema(args.schema)

    print("\n=== Top-level keys ===")
    print(list(sc.keys()))

    # 1) central_agent & sim period
    print("\n=== Simulation/Agent ===")
    print("central_agent:", sc.get('central_agent'))
    print("simulation_start_time_step:", sc.get('simulation_start_time_step'))
    print("simulation_end_time_step:", sc.get('simulation_end_time_step'))
    print("seconds_per_time_step:", sc.get('seconds_per_time_step'))

    # 2) observations（全局）
    obs = sc.get('observations', {})
    print("\n=== Global Observations ===")
    if not obs:
        print("⚠️  observations 是空的（这会导致 SAC 观测为 None）")
    else:
        actives = [k for k,v in obs.items() if isinstance(v, dict) and v.get('active', False)]
        inactives = [k for k,v in obs.items() if isinstance(v, dict) and not v.get('active', False)]
        print("active 数量:", len(actives))
        print("active 列表(前20):", actives[:20])
        if inactives:
            print("inactive 列表(前20):", inactives[:20])

    # 3) buildings（字典或列表）
    b = sc.get('buildings', {})
    print("\n=== Buildings ===")
    if isinstance(b, dict):
        b_names = list(b.keys())
        print("建筑数量:", len(b_names))
        sample = b_names[:5]
        print("示例建筑:", sample)
        # 检查 include / inactive_observations
        problem_buildings: List[str] = []
        fully_inactive: List[str] = []
        for name in b_names:
            cfg = b[name] or {}
            include = cfg.get('include', True)
            if not include:
                problem_buildings.append(name)
            inact = cfg.get('inactive_observations', [])
            # 如果把全局 active 都屏蔽了，也会导致该建筑 obs=None
            try:
                actives = [k for k,v in obs.items() if isinstance(v, dict) and v.get('active', False)]
                if actives and isinstance(inact, list) and set(inact) >= set(actives):
                    fully_inactive.append(name)
            except Exception:
                pass
        if problem_buildings:
            print(f"⚠️  有 {len(problem_buildings)} 栋建筑 include=false（可能导致 None）：", problem_buildings[:5])
        if fully_inactive:
            print(f"⚠️  有 {len(fully_inactive)} 栋建筑把所有全局观测都屏蔽了：", fully_inactive[:5])
    else:
        print("⚠️  buildings 不是字典（请贴内容我再看）")

    # 4) 快速“环境探针”（不训练，只 reset 看看 obs 形态）
    try:
        from citylearn.citylearn import CityLearnEnv
        ca = sc.get('central_agent', False)
        print("\n=== Env probe (reset) ===")
        env = CityLearnEnv(args.schema, central_agent=ca)
        obs0 = env.reset()
        print("reset() 返回类型:", type(obs0))
        if obs0 is None:
            print("❌ reset() 返回 None —— SAC 会在 FloatTensor(None) 处报错。")
        else:
            try:
                # centralized → 向量/数组；decentralized → list[数组]
                if isinstance(obs0, list):
                    shapes = [getattr(x, 'shape', None) for x in obs0]
                    print("分布式观测数量:", len(obs0), "示例形状:", shapes[:5])
                else:
                    print("集中式观测形状:", getattr(obs0, 'shape', None))
            except Exception:
                pass
        env.close()
    except Exception as e:
        print("（可选探针失败：）", e)

if __name__ == '__main__':
    main()

import numpy as np
import torch
from citylearn.agents.sac import SAC

class DebugSAC(SAC):
    """调试用 SAC：不改原库，但打印 observation & None 来源"""

    def get_post_exploration_prediction(self, observations, deterministic=False):

        # ✅ 如果是 dict，先转换为 list，避免原始错误
        if isinstance(observations, dict):
            observations = [obs for obs in observations.values()]

        actions = []

        for i, o in enumerate(observations):
            # ✅ Debug：打印原始 obs
            print(f"\n🔍 [Debug] Building {i} raw obs = {o}")

            if o is None:
                raise ValueError(f"❌ Observation from agent {i} is None (raw environment output)")

            # ✅ 编码
            encoded = self.get_encoded_observations(i, o)
            print(f"   ➤ Encoded obs = {encoded}")

            if encoded is None:
                raise ValueError(f"❌ get_encoded_observations() returned None for agent {i}")

            # ✅ 归一化
            normalized = self.get_normalized_observations(i, encoded)
            print(f"   ➤ Normalized obs = {normalized}")

            if normalized is None:
                raise ValueError(f"❌ get_normalized_observations() returned None for agent {i}")

            # ✅ 尝试转成 tensor
            try:
                o_tensor = torch.FloatTensor(normalized).unsqueeze(0).to(self.device)
            except Exception as e:
                print(f"❌ Failed to convert obs to tensor. Obs = {normalized}")
                raise e

            # ✅ 原来的 action 获取流程
            result = self.policy_net[i].sample(o_tensor)
            a = result[2] if deterministic else result[0]
            actions.append(a.detach().cpu().numpy()[0])

        return actions

# 初始化 normalization
obs_buffer = []

obs = env.reset()
for _ in range(500):  # 收集前 500 步
    action = env.action_space.sample()   # 随机动作
    obs, _, _, _ = env.step(action)
    obs_list = list(obs.values()) if isinstance(obs, dict) else obs
    obs_buffer.extend(obs_list)

obs_buffer = np.array(obs_buffer)
agent.obs_mean = np.mean(obs_buffer, axis=0)
agent.obs_std = np.std(obs_buffer, axis=0) + 1e-6  # 避免除0

print("✅ Normalization initialized.")

from citylearn.citylearn import CityLearnEnv
from citylearn.agents.sac import SAC
from pathlib import Path

def run_v2g(agent_class, schema, episodes=3, central_agent=False, save_dir=None, **kwargs):
    print(f"\n=== Running {agent_class.__name__} (Vehicle-to-Grid) ===")
    # ✅ 启用 include_evs / v2g 参数
    env = CityLearnEnv(schema, central_agent=central_agent, include_evs=True, **kwargs)
    
    agent = agent_class(env)
    agent.learn(episodes=episodes, deterministic_finish=True)
    results = env.evaluate()
    env.close()

    print(f"✅ {agent_class.__name__} (V2G) finished.")
    if save_dir:
        Path(save_dir).mkdir(parents=True, exist_ok=True)
        (Path(save_dir)/"metrics.txt").write_text(str(results))
    return results

def main():
    # ✅ 使用含 EV battery 的 V2G 版本 schema
    schema_v2g = "data/citylearn_challenge_2022/v2g_schema.json"

    # ✅ 运行 SAC 控制器（Vehicle-to-Grid 实验）
    res_v2g = run_v2g(SAC, schema_v2g, episodes=5, central_agent=False, save_dir="results/4_3_v2g_sac")

    print("\n=== V2G SAC Results ===")
    print(res_v2g)

if __name__ == "__main__":
    main()