First commit

.gitignore

@@ -162,3 +162,17 @@ cython_debug/
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
 
+
+
+# 3rdparty/
+# __MACOSX/
+# __pycache__/
+# results/
+# ./debug/
+# *.zip
+# .DS_Store
+# *.pyc
+# .idea/
+# ./install_dependencies.sh
+# ./install_sc2.sh
+

123.py

@@ -0,0 +1 @@
+print('hello')

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2017 Google Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,2 +1,257 @@
-# pymarl3
+# [pymarl3](https://github.com/tjuHaoXiaotian/pymarl3): the source code of the ICLR-2023 paper 
+**[Boosting Multi-Agent Reinforcement Learning via Permutation Invariant and Permutation Equivariant Networks](https://openreview.net/pdf?id=OxNQXyZK-K8)**.
+
+We extend [**pymarl2** (https://github.com/hijkzzz/pymarl2)](https://github.com/hijkzzz/pymarl2) to **pymarl3**, adding the support for the [SMAC-V2 environment](https://github.com/oxwhirl/smacv2)  and equipping the MARL algorithms with permutation invariance and permutation equivariance properties. 
+
+## Key Features:
+* (1) **Support both [SMAC-V1](https://github.com/oxwhirl/smac) and [SMAC-V2](https://github.com/oxwhirl/smacv2)** (without the need of installing each environment separately). 
+  * ![SMAC-V2 configs](./doc/figure/smac_v2_config.png)
+* (2) Equip the MARL algorithms of [**pymarl2**](https://github.com/hijkzzz/pymarl2) with the **permutation invariance (PI) and permutation equivariance (PE)** properties. The proposed PI and PE model architectures **can be easily plugged into any existing MARL algorithms and boost their performance**.
+* (3) :rocket: **The enhanced algorithm achieves State-Of-The-Art (SOTA) performance on SMAC-V1 and SMAC-V2** (without restricting the agent field-of-view and shooting range to a cone).
+
+```
+[2023-07 update]: Commit the support for SMAC-V2.
+```
+
+## 1. Model Architecture of Hyper Policy Network (HPN)
+
+![Agent permutation invariant network with hypernetworks](./doc/figure/API-HPN.png)
+
+HPN incorporates [hypernetworks](https://arxiv.org/pdf/1609.09106) to generate different
+weights ![](http://latex.codecogs.com/svg.latex?W_i)s for different input
+components ![](http://latex.codecogs.com/svg.latex?x_i)s to improve representational capacity while ensuring the
+same ![](http://latex.codecogs.com/svg.latex?x_i) always be assigned with the same
+weight ![](http://latex.codecogs.com/svg.latex?W_i). The architecture of our HPN is shown in the above Figure (b). We
+also take the ![](http://latex.codecogs.com/svg.latex?Q_i(o_i)) as an example. The model mainly composes of two modules:
+
+**Permutation Invariant Input Layer.**  [hypernetworks](https://arxiv.org/pdf/1609.09106) are a family of neural
+architectures which use one network, known as hypernetwork, to generate the weights for another network. In our setting,
+the hypernetwork is utilized to generate a different ![](http://latex.codecogs.com/svg.latex?W_i) for
+each ![](http://latex.codecogs.com/svg.latex?x_i) of the input set ![](http://latex.codecogs.com/svg.latex?X_j). As
+shown in above Figure (b), ![](http://latex.codecogs.com/svg.latex?X_j) (which can be viewed as a batch
+of ![](http://latex.codecogs.com/svg.latex?m) ![](http://latex.codecogs.com/svg.latex?x_i)s each of which is of
+dimension ![](http://latex.codecogs.com/svg.latex?k), represented by different shades of blue) is firstly fed into a
+shared hypernetwork (marked in yellow), whose input size is ![](http://latex.codecogs.com/svg.latex?k) and output size
+is ![](http://latex.codecogs.com/svg.latex?k*h). Then, the corresponding outputs are reshaped
+to ![](http://latex.codecogs.com/svg.latex?\[k,h\]) and serve as the submodule
+weights ![](http://latex.codecogs.com/svg.latex?W_i)s of the normal FC layer (see Figure (a)). Note that
+different ![](http://latex.codecogs.com/svg.latex?x_i)s will generate
+different ![](http://latex.codecogs.com/svg.latex?W_i)s and the same ![](http://latex.codecogs.com/svg.latex?x_i) will
+always correspond to the same ![](http://latex.codecogs.com/svg.latex?W_i). Then,
+each ![](http://latex.codecogs.com/svg.latex?x_i) is multiplied by ![](http://latex.codecogs.com/svg.latex?W_i) and all
+multiplication results and the bias ![](http://latex.codecogs.com/svg.latex?b) are summed together to get the output.
+Since each element ![](http://latex.codecogs.com/svg.latex?x_i) is processed separately by its
+corresponding ![](http://latex.codecogs.com/svg.latex?W_i) and then merged by a permutation invariant 'sum' function,
+the permutation invariance is reserved.
+
+**Permutation Equivariance Output Layer.** Similarly, to keep the whole network permutation equivariance, the submodular
+weights and bias of the agent-related actions in the output layer,
+e.g., ![](http://latex.codecogs.com/svg.latex?\mathcal{A}_i^\text{attack}) of SMAC, are also generated by a
+hypernetwork. As mentioned above, the input ![](http://latex.codecogs.com/svg.latex?x_i) and
+output ![](http://latex.codecogs.com/svg.latex?W_i) of the hypernetwork always correspond one-to-one, so the input order
+change will result in the same output order change, thus achieving permutation equivariance.
+
+We emphasize that HPN is a general design and can be easily integrated into existing MARL algorithms (
+e.g., [VDN](https://arxiv.org/pdf/1706.05296?ref=https://githubhelp.com)
+, [QMIX](http://proceedings.mlr.press/v80/rashid18a/rashid18a.pdf)
+, [MADDPG](https://proceedings.neurips.cc/paper/2017/file/68a9750337a418a86fe06c1991a1d64c-Paper.pdf)
+, [MAPPO](https://arxiv.org/pdf/2103.01955?ref=https://githubhelp.com)) to boost the learning speed as well as the
+converged performance. All parameters of HPN are simply trained end-to-end with backpropagation according to the
+corresponding RL loss function.
+
+## 2. Experimental Results on SAMC-V1
+
+We mainly evaluate our methods on the challenging StarCraft II micromanagement
+benchmark [(SMAC)](https://github.com/oxwhirl/smac).
+
+****
+
+```
+StarCraft 2 version: SC2.4.10. difficulty: 7.
+```
+
+| Senarios       | Difficulty |               HPN-QMIX              |
+|----------------|:----------:|:----------------------------------:|
+| 8m_vs_9m           |  Hard |          **100%**          |
+| 5m_vs_6m     |    Hard    |          **100%**          |
+| 3s_vs_5z     |    Hard    |          **100%**          |
+| bane_vs_bane |    Hard    |          **100%**          |
+| 2c_vs_64zg   |    Hard    |          **100%**          |
+| corridor       | Super Hard |          **100%**          |
+| MMM2           | Super Hard |          **100%**          |
+| 3s5z_vs_3s6z | Super Hard |**100%** |
+| 27m_vs_30m   | Super Hard |          **100%**          |
+| 6h_vs_8z     | Super Hard |  **98%**  |
+
+### 2.1 Applying HPN to fine-tuned VDN and QMIX.
+
+![Applying HPN to fine-tuned VDN and QMIX](./doc/figure/exp_comparison_with_SOTA2.png)
+
+### 2.2 Applying HPN to QPLEX and MAPPO.
+
+![Applying HPN to QPLEX](./doc/figure/HPN-QPLEX.png)
+![Applying HPN to MAPPO](./doc/figure/HPN-mappo.png)
+
+### 2.3 Comparison with baselines considering permutation invariance or permutation equivariance.
+
+![Comparison with Related Baselines](./doc/figure/exp_comparison_with_baselines.png)
+
+### 2.4 Transfer results.
+
+Apart from achieving PI and PE, another benefit of HPN is that it can naturally handle variable numbers of inputs and
+outputs. Therefore, as also stated in the conclusion section, HPN can be potentially used to design more efficient
+multitask learning and transfer learning algorithms. For example, we can directly transfer the learned HPN policy in one
+task to new tasks with different numbers of agents and improve the learning efficiency in the new tasks. Transfer
+learning results of 5m → 12m, 5m_vs_6m → 8m_vs_10m, 3s_vs_3z → 3s_vs_5z are shown in the following figures. We see that
+the previously trained HPN policies can serve as better initialization policies for new tasks.
+![Transfer previous policy v.s. learning from scratch](./doc/figure/MA_transfer.png)
+
+## 3. Experimental Results on SAMC-V2
+
+### 3.1 Changes of SMAC-V2.
+
+SMAC-v2 makes three major changes to SMAC: randomising start positions, randomising unit types, and restricting the
+agent field-of-view and shooting range to a cone. These first two changes increase more randomness to challenge
+contemporary MARL algorithms. The third change makes features harder to infer and adds the challenge that agents must
+actively gather information (require more efficient exploration). **Since our target is not to design more efficient
+exploration algorithms, we keep the field-of-view and attack of the agents a full circle as in SMAC-V1.**
+
+* **Random Start Positions:**  Random start positions come in two different types. First, there is the `surrounded`
+  type, where the allied units are spawned in the middle of the map, and surrounded by enemy units. This challenges the
+  allied units to overcome the enemies approach from multiple angles at once. Secondly, there are the `reflect_position`
+  scenarios. These randomly select positions for the allied units, and then reflect their positions in the midpoint of
+  the map to get the enemy spawn positions. Examples are shown in the figure
+  below.  ![Random Start Positions](./doc/figure/smac_v2_random_start_positions.png)
+* **Random Unit Types:**  Battles in SMAC-V2 do not always feature units of the same type each time, as they did in
+  SMAC. Instead, units are spawned randomly according to certain pre-fixed probabilities. Units in StarCraft II are
+  split up into different races. Units from different races cannot be on the same team. For each of the three races (
+  Protoss, Terran, and Zerg), SMAC-V2 uses three unit types. Detailed generation probabilities are shown in the figure
+  below. ![Random Start Positions](./doc/figure/smac_v2_random_unit_types.png)
+
+### 3.2 Experimental Results.
+
+**Our HPN can naturally handle the two types of new challenges.** Thanks to the PI and PE properties, our HPN is more
+robust to the randomly changed start positions of the entities. Thanks to the entity-wise modeling and using
+hypernetwork to generate a customized `weight matrix` for each type of unit, HPN can handle the randomly generated unit
+types as well. The comparisons of HPN-VDN with VDN on three difficult scenarios across the three races (Protoss, Terran,
+and Zerg) are shown in the figures below. Results show that our HPN significantly improves the sample efficiency and the
+converged test win rates of the baseline VDN.
+![HPN-VDN v.s. VDN](./doc/figure/smac_v2_results.png)
+
+## 4. How to use the code?
+
+### 4.1 Install this repository.
+```shell
+# Step-1: Clone the repository and enter the folder.
+git clone git@github.com:tjuHaoXiaotian/pymarl3.git
+cd pymarl3
+
+# Step-2: Install StarCraftII add the custom maps.
+chmod +x install_sc2.sh
+./install_sc2.sh
+
+# Step-3: Install PyTorch and other Python packages.
+chmod +x install_dependencies.sh
+./install_dependencies.sh
+```
+
+### 4.2 Detailed command lines to reproduce all experimental results (on SMAC-V1).
+
+```shell
+# For SMAC, take the hpn_qmix, qmix, hpn_qplex and qplex over all hard and super-hard scenarios for example.
+
+# 5m_vs_6m
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=hpn_qmix --env-config=sc2 with env_args.map_name=5m_vs_6m obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=qmix --env-config=sc2 with env_args.map_name=5m_vs_6m obs_agent_id=True obs_last_action=True runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=hpn_qplex --env-config=sc2 with env_args.map_name=5m_vs_6m obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=qplex --env-config=sc2 with env_args.map_name=5m_vs_6m obs_agent_id=True obs_last_action=True runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+
+# 3s5z_vs_3s6z
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=hpn_qmix --env-config=sc2 with env_args.map_name=3s5z_vs_3s6z obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=4 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=qmix --env-config=sc2 with env_args.map_name=3s5z_vs_3s6z obs_agent_id=True obs_last_action=True runner=parallel batch_size_run=4 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=hpn_qplex --env-config=sc2 with env_args.map_name=3s5z_vs_3s6z obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=4 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=qplex --env-config=sc2 with env_args.map_name=3s5z_vs_3s6z obs_agent_id=True obs_last_action=True runner=parallel batch_size_run=4 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+
+# 6h_vs_8z
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=hpn_qmix --env-config=sc2 with env_args.map_name=6h_vs_8z obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=500000 batch_size=128 td_lambda=0.3 hpn_head_num=2
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=qmix --env-config=sc2 with env_args.map_name=6h_vs_8z obs_agent_id=True obs_last_action=True runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=500000 batch_size=128 td_lambda=0.3 hpn_head_num=2
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=hpn_qplex --env-config=sc2 with env_args.map_name=6h_vs_8z obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=500000 batch_size=128 td_lambda=0.3 hpn_head_num=2
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=qplex --env-config=sc2 with env_args.map_name=6h_vs_8z obs_agent_id=True obs_last_action=True runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=500000 batch_size=128 td_lambda=0.3 hpn_head_num=2
+
+# 8m_vs_9m
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=hpn_qmix --env-config=sc2 with env_args.map_name=8m_vs_9m obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=qmix --env-config=sc2 with env_args.map_name=8m_vs_9m obs_agent_id=True obs_last_action=True runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=hpn_qplex --env-config=sc2 with env_args.map_name=8m_vs_9m obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=qplex --env-config=sc2 with env_args.map_name=8m_vs_9m obs_agent_id=True obs_last_action=True runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+
+# 3s_vs_5z
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=hpn_qmix --env-config=sc2 with env_args.map_name=3s_vs_5z obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6 hpn_head_num=2
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=qmix --env-config=sc2 with env_args.map_name=3s_vs_5z obs_agent_id=True obs_last_action=True runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6 hpn_head_num=2
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=hpn_qplex --env-config=sc2 with env_args.map_name=3s_vs_5z obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6 hpn_head_num=2
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=qplex --env-config=sc2 with env_args.map_name=3s_vs_5z obs_agent_id=True obs_last_action=True runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6 hpn_head_num=2
+
+# corridor
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=hpn_qmix --env-config=sc2 with env_args.map_name=corridor obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=qmix --env-config=sc2 with env_args.map_name=corridor obs_agent_id=True obs_last_action=True runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=hpn_qplex --env-config=sc2 with env_args.map_name=corridor obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=qplex --env-config=sc2 with env_args.map_name=corridor obs_agent_id=True obs_last_action=True runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+
+# MMM2
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=hpn_qmix --env-config=sc2 with env_args.map_name=MMM2 obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=qmix --env-config=sc2 with env_args.map_name=MMM2 obs_agent_id=True obs_last_action=True runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=hpn_qplex --env-config=sc2 with env_args.map_name=MMM2 obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=qplex --env-config=sc2 with env_args.map_name=MMM2 obs_agent_id=True obs_last_action=True runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+
+# 27m_vs_30m
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=hpn_qmix --env-config=sc2 with env_args.map_name=27m_vs_30m obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=qmix --env-config=sc2 with env_args.map_name=27m_vs_30m obs_agent_id=True obs_last_action=True runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=hpn_qplex --env-config=sc2 with env_args.map_name=27m_vs_30m obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=qplex --env-config=sc2 with env_args.map_name=27m_vs_30m obs_agent_id=True obs_last_action=True runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+
+# 2c_vs_64zg
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=hpn_qmix --env-config=sc2 with env_args.map_name=2c_vs_64zg obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=qmix --env-config=sc2 with env_args.map_name=2c_vs_64zg obs_agent_id=True obs_last_action=True runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=hpn_qplex --env-config=sc2 with env_args.map_name=2c_vs_64zg obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=qplex --env-config=sc2 with env_args.map_name=2c_vs_64zg obs_agent_id=True obs_last_action=True runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+
+# bane_vs_bane
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=hpn_qmix --env-config=sc2 with env_args.map_name=bane_vs_bane obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=qmix --env-config=sc2 with env_args.map_name=bane_vs_bane obs_agent_id=True obs_last_action=True runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=hpn_qplex --env-config=sc2 with env_args.map_name=bane_vs_bane obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=qplex --env-config=sc2 with env_args.map_name=bane_vs_bane obs_agent_id=True obs_last_action=True runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+```
+
+### 4.3 Detailed command lines to reproduce the experimental results (on SMAC-V2).
+
+```shell
+#%%%%%%%%%%%%%%%%%%% sc2_v2_terran %%%%%%%%%%%%%%%%%%%%%
+CUDA_VISIBLE_DEVICES="1" python src/main.py --config=hpn_vdn --env-config=sc2_v2_terran with obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6 mixer=vdn
+
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=vdn --env-config=sc2_v2_terran with obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6 mixer=vdn
+
+#%%%%%%%%%%%%%%%%%%% sc2_v2_protoss %%%%%%%%%%%%%%%%%%%%%
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=hpn_vdn --env-config=sc2_v2_protoss with obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6 mixer=vdn
+
+CUDA_VISIBLE_DEVICES="1" python src/main.py --config=vdn --env-config=sc2_v2_protoss with obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6 mixer=vdn
+
+#%%%%%%%%%%%%%%%%%%% sc2_v2_zerg %%%%%%%%%%%%%%%%%%%%%
+CUDA_VISIBLE_DEVICES="1" python src/main.py --config=hpn_vdn --env-config=sc2_v2_zerg with obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6 mixer=vdn
+
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=vdn --env-config=sc2_v2_zerg with obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6 mixer=vdn
+
+```
+
+## 5. Citation
+
+```text
+@article{hao2022api,
+  title={Boosting Multi-Agent Reinforcement Learning via Permutation Invariant and Permutation Equivariant Networks},
+  author={Hao Xiaotian, Hao Jianye, Mao Hangyu, Wang Weixun, Yang Yaodong, Li Dong, Zheng Yan, Wang Zhen},
+  journal={The Eleventh International Conference on Learning Representations.},
+  year={2023}
+}
+```
+
+
+
 

doc/README(old-version).md

@@ -0,0 +1,129 @@
+# API: Boosting Multi-Agent Reinforcement Learning via Agent-Permutation-Invariant Networks
+
+Open-source code for [API: Boosting Multi-Agent Reinforcement Learning via Agent-Permutation-Invariant Networks](https://arxiv.org/abs/xxxxx).
+
+[TOC]
+
+## 1. Motivation
+
+### 1.1  Permutation Invariance and Equivariance
+
+
+
+**Permutation Invariant Function.** A function ![](http://latex.codecogs.com/svg.latex?f:X\rightarrow\text{Y}) where ![](http://latex.codecogs.com/svg.latex?X=\left[x_1,x_2,\ldots\text{}x_m\right]^\mathsf{T}) of size ![](http://latex.codecogs.com/svg.latex?\(m,k\)) is a set consisting of ![](http://latex.codecogs.com/svg.latex?m) components (each of which is of dimension ![](http://latex.codecogs.com/svg.latex?k)), is said to be permutation invariant if permutation of input components does not change the output of the function. Mathematically, ![](http://latex.codecogs.com/svg.latex?f(\left[x_1,x_2,\ldots\text{}x_m\right]^\mathsf{T})=f(M\left[x_1,x_2,\ldots\text{}x_m\right]^\mathsf{T})), where ![](http://latex.codecogs.com/svg.latex?M) is the permutation matrix of size ![](http://latex.codecogs.com/svg.latex?\(m,m\)), which is a binary matrix that has exactly a single unit value in every row and column and zeros everywhere else.
+
+**Permutation Equivariant Function.** Similarly, a function ![](http://latex.codecogs.com/svg.latex?f:X\rightarrow\text{Y}) is permutation equivariant if permutation of input components permutes the output components with the same permutation ![](http://latex.codecogs.com/svg.latex?M). Mathematically, ![](http://latex.codecogs.com/svg.latex?f(M\left[x_1,x_2,\ldots\text{}x_m\right]^\mathsf{T})=M\left[y_1,y_2,\ldots\text{}y_m\right]^\mathsf{T}).
+
+### 1.2 Why Permutation Invariant Matters?
+
+In MARL, the environments typically consist of ![](http://latex.codecogs.com/svg.latex?m) components, including ![](http://latex.codecogs.com/svg.latex?n) learning agents and ![](http://latex.codecogs.com/svg.latex?m-n) non-player characters. Therefore, the states, observations are factorizable as sets of ![](http://latex.codecogs.com/svg.latex?m) components ![](http://latex.codecogs.com/svg.latex?\left\[x_1,\ldots\text{}x_m\right\]), where each component ![](http://latex.codecogs.com/svg.latex?x_i) represents an atomic semantic meaning (e.g., agent ![](http://latex.codecogs.com/svg.latex?i)'s features) whose dimension is ![](http://latex.codecogs.com/svg.latex?k). Because shuffling the order of ![](http://latex.codecogs.com/svg.latex?m) components does not change the information of the set, one would expect many functions, e.g., the policy function ![](http://latex.codecogs.com/svg.latex?\pi_i(a_i|o_i)), possess permutation invariance and permutation equivariance. These properties can be exploited to design more efficient MARL algorithms, especially when the ![](http://latex.codecogs.com/svg.latex?m) components are homogeneous, i.e., semantically identical (belonging to the same type, having identical feature spaces, action spaces and reward functions).
+
+Taking ![](http://latex.codecogs.com/svg.latex?Q_i(a_i|o_i)) as an example, the input is the observation ![](http://latex.codecogs.com/svg.latex?o_i=\left[x_1,\ldots\text{}x_m\right]), and the outputs are Q-values of all actions in ![](http://latex.codecogs.com/svg.latex?\mathcal{A}_i). Since the ![](http://latex.codecogs.com/svg.latex?m) components are homogeneous, they have the same feature space, i.e., ![](http://latex.codecogs.com/svg.latex?\forall\text{}x_i\in\mathcal{X}). Thus, the size of an **fixedly ordered representation** of ![](http://latex.codecogs.com/svg.latex?o_i) is ![](http://latex.codecogs.com/svg.latex?|\mathcal{X}|^m). In contrast, using a **permutation invariant representation**, i.e., removing the influence of the input order, could reduce the size of the observation space by a factor of ![](http://latex.codecogs.com/svg.latex?\frac{1}{m!}). As the number of homogeneous components increases, the removal of these redundancies results in a much smaller search space, upon which we could more easily learn a policy.
+
+Our objective is to design more flexible **Agent Permutation Invariant** (**API**) and **Agent Permutation Equivariant** (**APE**) models to greatly reduce the sample complexity of MARL algorithms. Also taking ![](http://latex.codecogs.com/svg.latex?Q_i(a_i|o_i)) as the example, if there is a direct correspondence between the action Q-value in output and the component in input ![](http://latex.codecogs.com/svg.latex?o_i), then ![](http://latex.codecogs.com/svg.latex?Q_i(a_i|o_i)) for these actions should be permutation equivariant; otherwise, ![](http://latex.codecogs.com/svg.latex?Q_i(a_i|o_i)) should be permutation invariant.
+
+<img src="./figure/API_APE_function.png" alt="API_APE_function" style="zoom:50%;" />
+
+Note that this is very common for many multi-agent settings. For example, as illustrated in the above Figure, in the challenging [StarCraft II micromanagement benchmark (SMAC)](https://github.com/oxwhirl/smac), the input set ![](http://latex.codecogs.com/svg.latex?o_i=\left[x_1,\ldots\text{}x_m\right]) could be divided into 2 groups: an ally group ![](http://latex.codecogs.com/svg.latex?o_i^{\text{ally}}) and an enemy group ![](http://latex.codecogs.com/svg.latex?o_i^{\text{enemy}}). The output Q-values of the actions could be divided into 2 groups as well: Q-values for move actions ![](http://latex.codecogs.com/svg.latex?\mathcal{A}_i^\text{move}), i.e., ![](http://latex.codecogs.com/svg.latex?\left[\text{noop},\text{stop},\uparrow,\rightarrow,\downarrow,\leftarrow\right]), and attack actions ![](http://latex.codecogs.com/svg.latex?\mathcal{A}_i^\text{attack}). Since there is a one-to-one correspondence between the elements in ![](http://latex.codecogs.com/svg.latex?o_i^{\text{enemy}}) and ![](http://latex.codecogs.com/svg.latex?\mathcal{A}_i^\text{attack}), the Q-values of ![](http://latex.codecogs.com/svg.latex?\mathcal{A}_i^\text{attack}) should be equivariant to the permutations of ![](http://latex.codecogs.com/svg.latex?o_i^{\text{enemy}}), while the Q-values of ![](http://latex.codecogs.com/svg.latex?\mathcal{A}_i^\text{move}) should be invariant to the permutations of the whole set ![](http://latex.codecogs.com/svg.latex?o_i). Overall, a desired model of ![](http://latex.codecogs.com/svg.latex?Q_i(o_i)) should be both permutation invariant and permutation equivariance.
+
+
+
+## 2. Model Architecture of API-HyPerNetwork (API-HPN)
+
+![Agent permutation invariant network with hypernetworks](./figure/API-HPN.png)
+
+API-HPN incorporates [hypernetworks](https://arxiv.org/pdf/1609.09106) to generate different weights ![](http://latex.codecogs.com/svg.latex?W_i)s for different input components ![](http://latex.codecogs.com/svg.latex?x_i)s to improve representational capacity while ensuring the same ![](http://latex.codecogs.com/svg.latex?x_i) always be assigned with the same weight ![](http://latex.codecogs.com/svg.latex?W_i). The architecture of our API-HPN is shown in the above Figure (b). We also take the ![](http://latex.codecogs.com/svg.latex?Q_i(o_i)) as an example. The model mainly composes of two modules:
+
+**Agent Permutation Invariant Input Layer.**  [hypernetworks](https://arxiv.org/pdf/1609.09106) are a family of neural architectures which use one network, known as hypernetwork, to generate the weights for another network. In our setting, the hypernetwork is utilized to generate a different ![](http://latex.codecogs.com/svg.latex?W_i) for each ![](http://latex.codecogs.com/svg.latex?x_i) of the input set ![](http://latex.codecogs.com/svg.latex?X_j). As shown in above Figure (b), ![](http://latex.codecogs.com/svg.latex?X_j) (which can be viewed as a batch of ![](http://latex.codecogs.com/svg.latex?m) ![](http://latex.codecogs.com/svg.latex?x_i)s each of which is of dimension ![](http://latex.codecogs.com/svg.latex?k), represented by different shades of blue) is firstly fed into a shared hypernetwork (marked in yellow), whose input size is ![](http://latex.codecogs.com/svg.latex?k) and output size is ![](http://latex.codecogs.com/svg.latex?k*h). Then, the corresponding outputs are reshaped to ![](http://latex.codecogs.com/svg.latex?\[k,h\]) and serve as the submodule weights ![](http://latex.codecogs.com/svg.latex?W_i)s of the normal FC layer (see Figure (a)). Note that different ![](http://latex.codecogs.com/svg.latex?x_i)s will generate different ![](http://latex.codecogs.com/svg.latex?W_i)s and the same ![](http://latex.codecogs.com/svg.latex?x_i) will always correspond to the same ![](http://latex.codecogs.com/svg.latex?W_i). Then, each ![](http://latex.codecogs.com/svg.latex?x_i) is multiplied by ![](http://latex.codecogs.com/svg.latex?W_i) and all multiplication results and the bias ![](http://latex.codecogs.com/svg.latex?b) are summed together to get the output. Since each element ![](http://latex.codecogs.com/svg.latex?x_i) is processed separately by its corresponding ![](http://latex.codecogs.com/svg.latex?W_i) and then merged by a permutation invariant 'sum' function, the permutation invariance is reserved.
+
+**Agent Permutation Equivariance Output Layer.** Similarly, to keep the whole network permutation equivariance, the submodular weights and bias of the agent-related actions in the output layer, e.g., ![](http://latex.codecogs.com/svg.latex?\mathcal{A}_i^\text{attack}) of SMAC, are also generated by a hypernetwork. As mentioned above, the input ![](http://latex.codecogs.com/svg.latex?x_i) and output ![](http://latex.codecogs.com/svg.latex?W_i) of the hypernetwork always correspond one-to-one, so the input order change will result in the same output order change, thus achieving permutation equivariance.
+
+We emphasize that API-HPN is a general design and can be easily integrated into existing MARL algorithms (e.g., [VDN](https://arxiv.org/pdf/1706.05296?ref=https://githubhelp.com), [QMIX](http://proceedings.mlr.press/v80/rashid18a/rashid18a.pdf), [MADDPG](https://proceedings.neurips.cc/paper/2017/file/68a9750337a418a86fe06c1991a1d64c-Paper.pdf), [MAPPO](https://arxiv.org/pdf/2103.01955?ref=https://githubhelp.com)) to boost the learning speed as well as the converged performance. All parameters of API-HPN are simply trained end-to-end with backpropagation according to the corresponding RL loss function.
+
+
+
+## 3. Experiments
+
+### 3.1 Experimental Setups
+
+We mainly evaluate our methods in the challenging StarCraft II micromanagement benchmark [(SMAC)](https://github.com/oxwhirl/smac).
+
+****
+
+```
+StarCraft 2 version: SC2.4.10. difficulty: 7.
+```
+### 3.2 Evaluation Metric
+
+### 3.3 Code Implementations and Structure
+
+### 3.4 Results
+
+#### 3.4.1 Comparison with previous SOTA
+
+![The Full Comparison of API-HPN with SOTA on SMAC](./figure/exp_comparison_with_SOTA.png)
+
+#### 3.4.2 Comparison with baselines considering permutation invariance and permutation equivariant property
+
+![Comparison with Related Baselines](./figure/exp_comparison_with_baselines.png)
+
+#### 3.4.3 Ablation Studies
+
+![ablation](./figure/exp_ablation.png)
+
+
+
+
+| Senarios       | Difficulty |               API-QMIX              |
+|----------------|:----------:|:----------------------------------:|
+| 8m_vs_9m           |  Hard |          **100%**          |
+| 5m_vs_6m     |    Hard    |          **100%**          |
+| 3s_vs_5z     |    Hard    |          **100%**          |
+| bane_vs_bane |    Hard    |          **100%**          |
+| 2c_vs_64zg   |    Hard    |          **100%**          |
+| corridor       | Super Hard |          **100%**          |
+| MMM2           | Super Hard |          **100%**          |
+| 3s5z_vs_3s6z | Super Hard |**100%** |
+| 27m_vs_30m   | Super Hard |          **100%**          |
+| 6h_vs_8z     | Super Hard |  **98%**  |
+
+
+
+## 4. How to use the code?
+
+### 4.1 Detailed Command line tool to reproduce all experimental results
+
+
+
+**Run an experiment**
+
+```shell
+# For SMAC, take the 5m_vs_6m scenario for example.
+CUDA_VISIBLE_DEVICES="0" python src/main.py --config=api_vdn --env-config=sc2 with env_args.map_name=5m_vs_6m obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+
+CUDA_VISIBLE_DEVICES="1" python src/main.py --config=api_qmix --env-config=sc2 with env_args.map_name=5m_vs_6m obs_agent_id=True obs_last_action=False runner=parallel batch_size_run=8 buffer_size=5000 t_max=10050000 epsilon_anneal_time=100000 batch_size=128 td_lambda=0.6
+```
+
+
+
+
+The config files act as defaults for an algorithm or environment. 
+
+They are all located in `src/config`.
+`--config` refers to the config files in `src/config/algs`
+`--env-config` refers to the config files in `src/config/envs`
+
+
+
+# Citation
+```
+@article{,
+      title={API: Boosting Multi-Agent Reinforcement Learning via Agent-Permutation-Invariant Networks}, 
+      author={},
+      year={2022},
+      eprint={},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG}
+}
+```
+

install_dependencies.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+# Install PyTorch and Python Packages
+# 3. Install Python dependencies
+echo 'Install PyTorch and Python dependencies...'
+# conda create -n pymarl python=3.8 -y
+# conda activate pymarl
+
+conda install pytorch torchvision torchaudio cudatoolkit=11.1 -c pytorch-lts -c nvidia -y
+pip install sacred numpy scipy gym==0.10.8 matplotlib seaborn \
+    pyyaml==5.3.1 pygame pytest probscale imageio snakeviz tensorboard-logger
+
+# pip install git+https://github.com/oxwhirl/smac.git
+# Do not need install SMAC anymore. We have integrated SMAC-V1 and SMAC-V2 in pymarl3/envs.
+pip install "protobuf<3.21"
+pip install "pysc2>=3.0.0"
+pip install "s2clientprotocol>=4.10.1.75800.0"
+pip install "absl-py>=0.1.0"

install_sc2.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# Install SC2 and add the custom maps
+
+# Clone the source code.
+#git clone git@github.com:tjuHaoXiaotian/pymarl3.git
+export PYMARL3_CODE_DIR=$(pwd)
+
+# 1. Install StarCraftII
+echo 'Install StarCraftII...'
+cd "$HOME"
+export SC2PATH="$HOME/StarCraftII"
+echo 'SC2PATH is set to '$SC2PATH
+if [ ! -d $SC2PATH ]; then
+        echo 'StarCraftII is not installed. Installing now ...';
+        wget http://blzdistsc2-a.akamaihd.net/Linux/SC2.4.10.zip
+        unzip -P iagreetotheeula SC2.4.10.zip
+else
+        echo 'StarCraftII is already installed.'
+fi
+
+# 2. Install the custom maps
+
+# Copy the maps to the target dir.
+echo 'Install SMACV1 and SMACV2 maps...'
+MAP_DIR="$SC2PATH/Maps/"
+if [ ! -d "$MAP_DIR/SMAC_Maps" ]; then
+    echo 'MAP_DIR is set to '$MAP_DIR
+    if [ ! -d $MAP_DIR ]; then
+            mkdir -p $MAP_DIR
+    fi
+    cp -r "$PYMARL3_CODE_DIR/src/envs/smac_v2/official/maps/SMAC_Maps" $MAP_DIR
+else
+    echo 'SMACV1 and SMACV2 maps are already installed.'
+fi
+echo 'StarCraft II and SMAC maps are installed.'

obs_config.json

@@ -0,0 +1,329 @@
+{
+    "2c_vs_64zg_obs" : {
+        "model_input_size": 404, 
+        "env_obs_size": 332, 
+        "n_agent": 2, 
+        "n_enemy": 64, 
+        "model_input_compose": {
+            "0": {"name": "move_feats", "size": 4}, 
+            "1": {"name": "enemy_feats", "size": [64, 5]}, 
+            "2": {"name": "ally_feats", "size": [1, 6]}, 
+            "3": {"name": "own_feats", "size": 2}, 
+            "4": {"name": "last_action", "size": 70}, 
+            "5": {"name": "agent_id", "size": 2}
+        }
+    },
+
+    "2m_vs_1z_obs" : {
+        "model_input_size": 25, 
+        "env_obs_size": 16, 
+        "n_agent": 2, 
+        "n_enemy": 1, 
+        "model_input_compose": {
+            "0": {"name": "move_feats", "size": 4}, 
+            "1": {"name": "enemy_feats", "size": [1, 6]}, 
+            "2": {"name": "ally_feats", "size": [1, 5]}, 
+            "3": {"name": "own_feats", "size": 1}, 
+            "4": {"name": "last_action", "size": 7}, 
+            "5": {"name": "agent_id", "size": 2}
+        }
+    },
+
+    "2s_vs_1sc_obs" : {
+        "model_input_size": 26, 
+        "env_obs_size": 17, 
+        "n_agent": 2, 
+        "n_enemy": 1, 
+        "model_input_compose": {
+            "0": {"name": "move_feats", "size": 4}, 
+            "1": {"name": "enemy_feats", "size": [1, 5]}, 
+            "2": {"name": "ally_feats", "size": [1, 6]}, 
+            "3": {"name": "own_feats", "size": 2}, 
+            "4": {"name": "last_action", "size": 7}, 
+            "5": {"name": "agent_id", "size": 2}
+        }
+    },
+
+    "2s3z_obs" : {
+        "model_input_size": 96, 
+        "env_obs_size": 80, 
+        "n_agent": 5, 
+        "n_enemy": 5, 
+        "model_input_compose": {
+            "0": {"name": "move_feats", "size": 4}, 
+            "1": {"name": "enemy_feats", "size": [5, 8]}, 
+            "2": {"name": "ally_feats", "size": [4, 8]}, 
+            "3": {"name": "own_feats", "size": 4}, 
+            "4": {"name": "last_action", "size": 11}, 
+            "5": {"name": "agent_id", "size": 5}
+        }
+    },
+
+    "3m_obs" : {
+        "model_input_size": 42, 
+        "env_obs_size": 30, 
+        "n_agent": 3, 
+        "n_enemy": 3, 
+        "model_input_compose": {
+            "0": {"name": "move_feats", "size": 4}, 
+            "1": {"name": "enemy_feats", "size": [3, 5]}, 
+            "2": {"name": "ally_feats", "size": [2, 5]}, 
+            "3": {"name": "own_feats", "size": 1}, 
+            "4": {"name": "last_action", "size": 9}, 
+            "5": {"name": "agent_id", "size": 3}
+        }
+    },
+
+    "3s_vs_3z_obs" : {
+        "model_input_size": 48, 
+        "env_obs_size": 36, 
+        "n_agent": 3, 
+        "n_enemy": 3, 
+        "model_input_compose": {
+            "0": {"name": "move_feats", "size": 4}, 
+            "1": {"name": "enemy_feats", "size": [3, 6]}, 
+            "2": {"name": "ally_feats", "size": [2, 6]}, 
+            "3": {"name": "own_feats", "size": 2}, 
+            "4": {"name": "last_action", "size": 9}, 
+            "5": {"name": "agent_id", "size": 3}
+        }
+    },
+
+    "3s_vs_4z_obs" : {
+        "model_input_size": 55, 
+        "env_obs_size": 42, 
+        "n_agent": 3, 
+        "n_enemy": 4, 
+        "model_input_compose": {
+            "0": {"name": "move_feats", "size": 4}, 
+            "1": {"name": "enemy_feats", "size": [4, 6]}, 
+            "2": {"name": "ally_feats", "size": [2, 6]}, 
+            "3": {"name": "own_feats", "size": 2}, 
+            "4": {"name": "last_action", "size": 10}, 
+            "5": {"name": "agent_id", "size": 3}
+        }
+    },
+
+    "3s_vs_5z_obs" : {
+        "model_input_size": 62, 
+        "env_obs_size": 48, 
+        "n_agent": 3, 
+        "n_enemy": 5, 
+        "model_input_compose": {
+            "0": {"name": "move_feats", "size": 4}, 
+            "1": {"name": "enemy_feats", "size": [5, 6]}, 
+            "2": {"name": "ally_feats", "size": [2, 6]}, 
+            "3": {"name": "own_feats", "size": 2}, 
+            "4": {"name": "last_action", "size": 11}, 
+            "5": {"name": "agent_id", "size": 3}
+        }
+    },
+
+    "3s5z_obs" : {
+        "model_input_size": 150, 
+        "env_obs_size": 128,
+        "n_agent": 8, 
+        "n_enemy": 8, 
+        "model_input_compose": {
+            "0": {"name": "move_feats", "size": 4}, 
+            "1": {"name": "enemy_feats", "size": [8, 8]}, 
+            "2": {"name": "ally_feats", "size": [7, 8]}, 
+            "3": {"name": "own_feats", "size": 4}, 
+            "4": {"name": "last_action", "size": 14}, 
+            "5": {"name": "agent_id", "size": 8}
+        }
+    },
+
+    "3s5z_vs_3s6z_obs" : {
+        "model_input_size": 159, 
+        "env_obs_size": 136, 
+        "n_agent": 8, 
+        "n_enemy": 9, 
+        "model_input_compose": {
+            "0": {"name": "move_feats", "size": 4}, 
+            "1": {"name": "enemy_feats", "size": [9, 8]}, 
+            "2": {"name": "ally_feats", "size": [7, 8]}, 
+            "3": {"name": "own_feats", "size": 4}, 
+            "4": {"name": "last_action", "size": 15}, 
+            "5": {"name": "agent_id", "size": 8}
+        }
+    },
+
+    "5m_vs_6m_obs" : {
+        "model_input_size": 72, 
+        "env_obs_size": 55, 
+        "n_agent": 5, "n_enemy": 6, 
+        "model_input_compose": {
+            "0": {"name": "move_feats", "size": 4}, 
+            "1": {"name": "enemy_feats", "size": [6, 5]}, 
+            "2": {"name": "ally_feats", "size": [4, 5]}, 
+            "3": {"name": "own_feats", "size": 1}, 
+            "4": {"name": "last_action", "size": 12}, 
+            "5": {"name": "agent_id", "size": 5}}
+    },
+
+    "6h_vs_8z_obs" : {
+        "model_input_size": 98, 
+        "env_obs_size": 78, 
+        "n_agent": 6, 
+        "n_enemy": 8, 
+        "model_input_compose": {
+            "0": {"name": "move_feats", "size": 4}, 
+            "1": {"name": "enemy_feats", "size": [8, 6]}, 
+            "2": {"name": "ally_feats", "size": [5, 5]}, 
+            "3": {"name": "own_feats", "size": 1}, 
+            "4": {"name": "last_action", "size": 14}, 
+            "5": {"name": "agent_id", "size": 6}
+        }
+    },
+
+    "8m_obs" : {
+        "model_input_size": 102, 
+        "env_obs_size": 80, 
+        "n_agent": 8, 
+        "n_enemy": 8, 
+        "model_input_compose": {
+            "0": {"name": "move_feats", "size": 4}, 
+            "1": {"name": "enemy_feats", "size": [8, 5]}, 
+            "2": {"name": "ally_feats", "size": [7, 5]}, 
+            "3": {"name": "own_feats", "size": 1}, 
+            "4": {"name": "last_action", "size": 14}, 
+            "5": {"name": "agent_id", "size": 8}
+        }
+    },
+
+    "8m_vs_9m_obs" : {
+        "model_input_size": 108, 
+        "env_obs_size": 85, 
+        "n_agent": 8, 
+        "n_enemy": 9, 
+        "model_input_compose": {
+            "0": {"name": "move_feats", "size": 4}, 
+            "1": {"name": "enemy_feats", "size": [9, 5]}, 
+            "2": {"name": "ally_feats", "size": [7, 5]}, 
+            "3": {"name": "own_feats", "size": 1}, 
+            "4": {"name": "last_action", "size": 15}, 
+            "5": {"name": "agent_id", "size": 8}
+        }
+    },
+
+    "10m_vs_11m_obs" : {
+        "model_input_size": 132, 
+        "env_obs_size": 105, 
+        "n_agent": 10, 
+        "n_enemy": 11, 
+        "model_input_compose": {
+            "0": {"name": "move_feats", "size": 4}, 
+            "1": {"name": "enemy_feats", "size": [11, 5]}, 
+            "2": {"name": "ally_feats", "size": [9, 5]}, 
+            "3": {"name": "own_feats", "size": 1}, 
+            "4": {"name": "last_action", "size": 17}, 
+            "5": {"name": "agent_id", "size": 10}
+        }
+    },
+
+    "25m_obs" : {
+        "model_input_size": 306, 
+        "env_obs_size": 250, 
+        "n_agent": 25, 
+        "n_enemy": 25, 
+        "model_input_compose": {
+            "0": {"name": "move_feats", "size": 4}, 
+            "1": {"name": "enemy_feats", "size": [25, 5]}, 
+            "2": {"name": "ally_feats", "size": [24, 5]}, 
+            "3": {"name": "own_feats", "size": 1}, 
+            "4": {"name": "last_action", "size": 31}, 
+            "5": {"name": "agent_id", "size": 25}
+        }
+    },
+
+    "27m_vs_30m_obs" : {
+        "model_input_size": 348, 
+        "env_obs_size": 285, 
+        "n_agent": 27, 
+        "n_enemy": 30, 
+        "model_input_compose": {
+            "0": {"name": "move_feats", "size": 4}, 
+            "1": {"name": "enemy_feats", "size": [30, 5]}, 
+            "2": {"name": "ally_feats", "size": [26, 5]}, 
+            "3": {"name": "own_feats", "size": 1}, 
+            "4": {"name": "last_action", "size": 36}, 
+            "5": {"name": "agent_id", "size": 27}
+        }
+    },
+
+    "bane_vs_bane_obs" : {
+        "model_input_size": 390, 
+        "env_obs_size": 336, 
+        "n_agent": 24, 
+        "n_enemy": 24, 
+        "model_input_compose": {
+            "0": {"name": "move_feats", "size": 4}, 
+            "1": {"name": "enemy_feats", "size": [24, 7]}, 
+            "2": {"name": "ally_feats", "size": [23, 7]}, 
+            "3": {"name": "own_feats", "size": 3}, 
+            "4": {"name": "last_action", "size": 30}, 
+            "5": {"name": "agent_id", "size": 24}
+        }
+    },
+
+    "corridor_obs" : {
+        "model_input_size": 192, 
+        "env_obs_size": 156, 
+        "n_agent": 6, 
+        "n_enemy": 24, 
+        "model_input_compose": {
+            "0": {"name": "move_feats", "size": 4}, 
+            "1": {"name": "enemy_feats", "size": [24, 5]}, 
+            "2": {"name": "ally_feats", "size": [5, 6]}, 
+            "3": {"name": "own_feats", "size": 2}, 
+            "4": {"name": "last_action", "size": 30}, 
+            "5": {"name": "agent_id", "size": 6}
+        }
+    },
+
+    "MMM_obs" : {
+        "model_input_size": 186, 
+        "env_obs_size": 160, 
+        "n_agent": 10, 
+        "n_enemy": 10, 
+        "model_input_compose": {
+            "0": {"name": "move_feats", "size": 4}, 
+            "1": {"name": "enemy_feats", "size": [10, 8]}, 
+            "2": {"name": "ally_feats", "size": [9, 8]}, 
+            "3": {"name": "own_feats", "size": 4}, 
+            "4": {"name": "last_action", "size": 16}, 
+            "5": {"name": "agent_id", "size": 10}
+        }
+    },
+
+    "MMM2_obs" : {
+        "model_input_size": 204, 
+        "env_obs_size": 176, 
+        "n_agent": 10, 
+        "n_enemy": 12, 
+        "model_input_compose": {
+            "0": {"name": "move_feats", "size": 4}, 
+            "1": {"name": "enemy_feats", "size": [12, 8]}, 
+            "2": {"name": "ally_feats", "size": [9, 8]}, 
+            "3": {"name": "own_feats", "size": 4}, 
+            "4": {"name": "last_action", "size": 18}, 
+            "5": {"name": "agent_id", "size": 10}
+        }
+    },
+
+    "so_many_baneling_obs" : {
+        "model_input_size": 247, 
+        "env_obs_size": 202, 
+        "n_agent": 7, 
+        "n_enemy": 32, 
+        "model_input_compose": {
+            "0": {"name": "move_feats", "size": 4}, 
+            "1": {"name": "enemy_feats", "size": [32, 5]}, 
+            "2": {"name": "ally_feats", "size": [6, 6]}, 
+            "3": {"name": "own_feats", "size": 2}, 
+            "4": {"name": "last_action", "size": 38}, 
+            "5": {"name": "agent_id", "size": 7}
+        }
+    }
+}

plotsmac.py

@@ -0,0 +1,117 @@
+import json
+import matplotlib.pyplot as plt
+import numpy as np
+import tkinter as tk
+from tkinter import ttk
+from scipy.signal import savgol_filter
+
+# 讀取多個 JSON 檔案
+def load_data(file_paths):
+    all_data = []
+    for file_path in file_paths:
+        with open(file_path) as f:
+            data = json.load(f)
+            # 處理特殊的數據格式（numpy.float64 對象）
+            processed_data = {}
+            for key, value in data.items():
+                if isinstance(value, list):
+                    # 檢查是否包含字典格式的數值
+                    if value and isinstance(value[0], dict) and 'value' in value[0]:
+                        processed_data[key] = [item['value'] for item in value]
+                    else:
+                        processed_data[key] = value
+                else:
+                    processed_data[key] = value
+            all_data.append(processed_data)
+    return all_data
+
+def smooth(y, window_length=51, polyorder=3):
+    return savgol_filter(y, window_length, polyorder)
+
+# 定義要繪製的數據
+def plot_data(data_list, keys, name_list, battle_name, smooth_window=2):
+    for key in keys:
+        # 創建一個圖形
+        fig, ax = plt.subplots()
+        # 設置坐標軸背景顏色
+        ax.set_facecolor('lightyellow')
+        # 設置網格顏色
+        ax.grid(color='green', linestyle='--', linewidth=0.5)
+        
+        for data, name in zip(data_list, name_list):
+            if key in data:
+                x = data[key + '_T']
+                y = data[key]
+                y_smooth = smooth(y)
+                ax.plot(x, y_smooth, label=name)
+                #ax.plot(data[key + '_T'], data[key], label=name) 
+        ax.set_xlabel('Time Steps')
+        ax.set_ylabel(key)
+        ax.set_title(battle_name)
+        ax.legend()
+
+    plt.show()
+
+def create_dynamic_window(data_list, keys, name_list, battle_name):
+    root = tk.Tk()
+    root.title("動態選擇要顯示的數據")
+    
+    # 創建左側的選擇面板
+    select_frame = ttk.Frame(root)
+    select_frame.pack(side=tk.LEFT, fill=tk.Y, padx=10, pady=5)
+    
+    # 創建變量來存儲選擇狀態
+    vars = []
+    
+    def update_plot(*args):
+        # 獲取選中的數據
+        selected_names = [name for name, var in zip(name_list, vars) if var.get()]
+        filtered_data = [data for data, name in zip(data_list, name_list) if name in selected_names]
+        filtered_names = [name for name in name_list if name in selected_names]
+        
+        # 清除所有現有的圖表
+        plt.close('all')
+        # 重新繪製圖表
+        if filtered_data:  # 確保至少選擇了一個數據
+            plot_data(filtered_data, keys, filtered_names, battle_name)
+    
+    # 創建複選框
+    for name in name_list:
+        var = tk.BooleanVar(value=True)  # 默認全選
+        var.trace('w', update_plot)  # 添加跟踪器，當值改變時更新圖表
+        vars.append(var)
+        cb = ttk.Checkbutton(select_frame, text=name, variable=var)
+        cb.pack(anchor='w', padx=5, pady=2)
+    
+    # 添加全選/取消全選按鈕
+    def select_all():
+        for var in vars:
+            var.set(True)
+    
+    def deselect_all():
+        for var in vars:
+            var.set(False)
+    
+    ttk.Button(select_frame, text="全選", command=select_all).pack(pady=5)
+    ttk.Button(select_frame, text="取消全選", command=deselect_all).pack(pady=5)
+    
+    # 初始繪圖
+    update_plot()
+    
+    root.mainloop()
+
+# 使用者選擇要繪製的數據
+file_paths = ['results/sacred/10gen_protoss/feudal/5/info.json',
+              'results/sacred/10gen_protoss/qmix/6/info.json',
+              ]
+
+data_list = load_data(file_paths)
+selected_keys = ['test_battle_won_mean','return_mean',"worker_loss","loss_td","manager_loss"]
+#selected_keys = ['battle_won_mean', 'loss']
+name_list = ['feudal', 'qmix'] 
+
+
+battle_name = '5protoss'
+
+# 使用新的動態選擇窗口
+create_dynamic_window(data_list, selected_keys, name_list, battle_name)

results/sacred/10gen_protoss/feudal/1/config.json

@@ -0,0 +1,131 @@
+{
+  "action_selector": "epsilon_greedy",
+  "agent": "feudal",
+  "agent_output_type": "q",
+  "asn_hidden_size": 32,
+  "batch_size": 128,
+  "batch_size_run": 4,
+  "buffer_cpu_only": true,
+  "buffer_size": 5000,
+  "c": 1,
+  "checkpoint_path": "",
+  "cpu_inference": true,
+  "critic_hidden_dim": 64,
+  "critic_lr": 0.0005,
+  "double_q": true,
+  "embedding_dim": 16,
+  "enable_parallel_computing": false,
+  "env": "sc2_v2",
+  "env_args": {
+    "capability_config": {
+      "n_units": 5,
+      "start_positions": {
+        "dist_type": "surrounded_and_reflect",
+        "map_x": 32,
+        "map_y": 32,
+        "n_enemies": 5,
+        "p": 0.5
+      },
+      "team_gen": {
+        "dist_type": "weighted_teams",
+        "exception_unit_types": [
+          "colossus"
+        ],
+        "observe": true,
+        "unit_types": [
+          "stalker",
+          "zealot",
+          "colossus"
+        ],
+        "weights": [
+          0.45,
+          0.45,
+          0.1
+        ]
+      }
+    },
+    "change_fov_with_move": false,
+    "conic_fov": false,
+    "continuing_episode": false,
+    "debug": false,
+    "difficulty": "7",
+    "fully_observable": false,
+    "game_version": null,
+    "heuristic_ai": false,
+    "heuristic_rest": false,
+    "kill_unit_step_mul": 2,
+    "map_name": "10gen_protoss",
+    "move_amount": 2,
+    "num_fov_actions": 12,
+    "obs_all_health": true,
+    "obs_instead_of_state": false,
+    "obs_last_action": false,
+    "obs_own_health": true,
+    "obs_own_pos": true,
+    "obs_pathing_grid": false,
+    "obs_starcraft": true,
+    "obs_terrain_height": false,
+    "obs_timestep_number": false,
+    "replay_dir": "",
+    "replay_prefix": "",
+    "reward_death_value": 10,
+    "reward_defeat": 0,
+    "reward_negative_scale": 0.5,
+    "reward_only_positive": true,
+    "reward_scale": true,
+    "reward_scale_rate": 20,
+    "reward_sparse": false,
+    "reward_win": 200,
+    "seed": null,
+    "state_last_action": true,
+    "state_timestep_number": false,
+    "step_mul": 8
+  },
+  "epsilon_anneal_time": 100000,
+  "epsilon_finish": 0.05,
+  "epsilon_start": 1.0,
+  "evaluate": false,
+  "gamma": 0.99,
+  "goal_dim": 16,
+  "grad_norm_clip": 10,
+  "hypernet_embed": 64,
+  "hypernet_layers": 2,
+  "intrinsic_rewards_alpha": 0.001,
+  "label": "default_label",
+  "learner": "feudal_learner",
+  "learner_log_interval": 10000,
+  "load_step": 0,
+  "local_results_path": "results",
+  "log_interval": 10000,
+  "lr": 0.0005,
+  "mac": "feudal_mac",
+  "manager_hidden_dim": 64,
+  "mixer": "qmix",
+  "mixing_embed_dim": 32,
+  "name": "feudal",
+  "obs_agent_id": true,
+  "obs_last_action": false,
+  "optim_alpha": 0.99,
+  "optim_eps": 1e-05,
+  "repeat_id": 1,
+  "rnn_hidden_dim": 64,
+  "run": "default",
+  "runner": "parallel",
+  "runner_log_interval": 10000,
+  "save_model": true,
+  "save_model_interval": 2000000,
+  "save_replay": false,
+  "seed": 618959174,
+  "state_dim": 80,
+  "t_max": 4050000,
+  "target_update_interval": 200,
+  "td_lambda": 0.6,
+  "test_greedy": true,
+  "test_interval": 10000,
+  "test_nepisode": 32,
+  "thread_num": 4,
+  "use_cuda": true,
+  "use_tensorboard": false,
+  "vf_coef": 0.5,
+  "worker_hidden_dim": 64
+}

results/sacred/10gen_protoss/feudal/1/info.json

@@ -0,0 +1,136 @@
+{
+  "battle_won_mean": [
+    0.0
+  ],
+  "battle_won_mean_T": [
+    216
+  ],
+  "dead_allies_mean": [
+    5.0
+  ],
+  "dead_allies_mean_T": [
+    216
+  ],
+  "dead_enemies_mean": [
+    0.0
+  ],
+  "dead_enemies_mean_T": [
+    216
+  ],
+  "ep_length_mean": [
+    54.0
+  ],
+  "ep_length_mean_T": [
+    216
+  ],
+  "epsilon": [
+    1.0
+  ],
+  "epsilon_T": [
+    216
+  ],
+  "return_max": [
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 4.99757281553398
+    }
+  ],
+  "return_max_T": [
+    216
+  ],
+  "return_mean": [
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 3.5837944867537175
+    }
+  ],
+  "return_mean_T": [
+    216
+  ],
+  "return_min": [
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 2.4034653465346536
+    }
+  ],
+  "return_min_T": [
+    216
+  ],
+  "return_std": [
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 1.0627956390471756
+    }
+  ],
+  "return_std_T": [
+    216
+  ],
+  "test_battle_won_mean": [
+    0.0
+  ],
+  "test_battle_won_mean_T": [
+    216
+  ],
+  "test_dead_allies_mean": [
+    5.0
+  ],
+  "test_dead_allies_mean_T": [
+    216
+  ],
+  "test_dead_enemies_mean": [
+    0.96875
+  ],
+  "test_dead_enemies_mean_T": [
+    216
+  ],
+  "test_ep_length_mean": [
+    54.5
+  ],
+  "test_ep_length_mean_T": [
+    216
+  ],
+  "test_return_max": [
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 12.280241935483874
+    }
+  ],
+  "test_return_max_T": [
+    216
+  ],
+  "test_return_mean": [
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 4.9681482554589795
+    }
+  ],
+  "test_return_mean_T": [
+    216
+  ],
+  "test_return_min": [
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 0.0
+    }
+  ],
+  "test_return_min_T": [
+    216
+  ],
+  "test_return_std": [
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 3.06453424806387
+    }
+  ],
+  "test_return_std_T": [
+    216
+  ]
+}

results/sacred/10gen_protoss/feudal/1/metrics.json

@@ -0,0 +1 @@
+{}

results/sacred/10gen_protoss/feudal/1/run.json

@@ -0,0 +1,125 @@
+{
+  "artifacts": [],
+  "command": "my_main",
+  "experiment": {
+    "base_dir": "C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src",
+    "dependencies": [
+      "numpy==1.23.1",
+      "PyYAML==6.0.2",
+      "sacred==0.8.7",
+      "torch==1.13.1+cu117"
+    ],
+    "mainfile": "main.py",
+    "name": "pymarl",
+    "repositories": [
+      {
+        "commit": "44bb188185fd0292a1a306c86396027efb44224d",
+        "dirty": true,
+        "url": "https://github.com/tjuHaoXiaotian/pymarl3.git"
+      },
+      {
+        "commit": "44bb188185fd0292a1a306c86396027efb44224d",
+        "dirty": true,
+        "url": "https://github.com/tjuHaoXiaotian/pymarl3.git"
+      }
+    ],
+    "sources": [
+      [
+        "main.py",
+        "_sources\\main_654daaa6534bcee62784d639ea63e51d.py"
+      ],
+      [
+        "utils\\logging.py",
+        "_sources\\logging_f71df6d788e929fac28afdf951d63d54.py"
+      ]
+    ]
+  },
+  "fail_trace": [
+    "Traceback (most recent call last):\n",
+    "  File \"C:\\Users\\Taiyo\\.conda\\envs\\SMACV2\\lib\\site-packages\\sacred\\config\\captured_function.py\", line 42, in captured_function\n    result = wrapped(*args, **kwargs)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\main.py\", line 44, in my_main\n    run_REGISTRY[_config['run']](_run, config, _log)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\run\\run.py\", line 128, in run\n    run_sequential(args=args, logger=logger)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\run\\run.py\", line 280, in run_sequential\n    learner.train(episode_sample, runner.t_env, episode)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\learners\\FeUdal_learner.py\", line 146, in train\n    cos_sim = F.cosine_similarity(temp_state_diff, temp_goals, dim=-1)\n",
+    "RuntimeError: The size of tensor a (57408) must match the size of tensor b (49920) at non-singleton dimension 0\n"
+  ],
+  "heartbeat": "2025-01-05T19:45:42.461491",
+  "host": {
+    "ENV": {},
+    "cpu": "AMD Ryzen 7 5700X3D 8-Core Processor",
+    "gpus": {
+      "driver_version": "560.94",
+      "gpus": [
+        {
+          "model": "NVIDIA GeForce RTX 4080 SUPER",
+          "persistence_mode": false,
+          "total_memory": 16376
+        }
+      ]
+    },
+    "hostname": "Taiyopen",
+    "os": [
+      "Windows",
+      "Windows-10-10.0.22631-SP0"
+    ],
+    "python_version": "3.10.16"
+  },
+  "meta": {
+    "command": "my_main",
+    "config_updates": {
+      "batch_size": 128,
+      "batch_size_run": 4,
+      "buffer_size": 5000,
+      "epsilon_anneal_time": 100000,
+      "obs_agent_id": true,
+      "obs_last_action": false,
+      "runner": "parallel",
+      "t_max": 4050000,
+      "td_lambda": 0.6,
+      "use_tensorboard": false
+    },
+    "named_configs": [],
+    "options": {
+      "--beat-interval": null,
+      "--capture": null,
+      "--comment": null,
+      "--debug": false,
+      "--enforce_clean": false,
+      "--file_storage": null,
+      "--force": false,
+      "--help": false,
+      "--id": null,
+      "--loglevel": null,
+      "--mongo_db": null,
+      "--name": null,
+      "--pdb": false,
+      "--print-config": false,
+      "--priority": null,
+      "--queue": false,
+      "--s3": null,
+      "--sql": null,
+      "--tiny_db": null,
+      "--unobserved": false,
+      "COMMAND": null,
+      "UPDATE": [
+        "obs_agent_id=True",
+        "obs_last_action=False",
+        "runner=parallel",
+        "batch_size_run=4",
+        "buffer_size=5000",
+        "t_max=4050000",
+        "epsilon_anneal_time=100000",
+        "batch_size=128",
+        "td_lambda=0.6",
+        "use_tensorboard=False"
+      ],
+      "help": false,
+      "with": true
+    }
+  },
+  "resources": [],
+  "result": null,
+  "start_time": "2025-01-05T19:44:34.041398",
+  "status": "FAILED",
+  "stop_time": "2025-01-05T19:45:42.465521"
+}

results/sacred/10gen_protoss/feudal/2/config.json

@@ -0,0 +1,131 @@
+{
+  "action_selector": "epsilon_greedy",
+  "agent": "feudal",
+  "agent_output_type": "q",
+  "asn_hidden_size": 32,
+  "batch_size": 128,
+  "batch_size_run": 4,
+  "buffer_cpu_only": true,
+  "buffer_size": 5000,
+  "c": 1,
+  "checkpoint_path": "",
+  "cpu_inference": true,
+  "critic_hidden_dim": 64,
+  "critic_lr": 0.0005,
+  "double_q": true,
+  "embedding_dim": 16,
+  "enable_parallel_computing": false,
+  "env": "sc2_v2",
+  "env_args": {
+    "capability_config": {
+      "n_units": 5,
+      "start_positions": {
+        "dist_type": "surrounded_and_reflect",
+        "map_x": 32,
+        "map_y": 32,
+        "n_enemies": 5,
+        "p": 0.5
+      },
+      "team_gen": {
+        "dist_type": "weighted_teams",
+        "exception_unit_types": [
+          "colossus"
+        ],
+        "observe": true,
+        "unit_types": [
+          "stalker",
+          "zealot",
+          "colossus"
+        ],
+        "weights": [
+          0.45,
+          0.45,
+          0.1
+        ]
+      }
+    },
+    "change_fov_with_move": false,
+    "conic_fov": false,
+    "continuing_episode": false,
+    "debug": false,
+    "difficulty": "7",
+    "fully_observable": false,
+    "game_version": null,
+    "heuristic_ai": false,
+    "heuristic_rest": false,
+    "kill_unit_step_mul": 2,
+    "map_name": "10gen_protoss",
+    "move_amount": 2,
+    "num_fov_actions": 12,
+    "obs_all_health": true,
+    "obs_instead_of_state": false,
+    "obs_last_action": false,
+    "obs_own_health": true,
+    "obs_own_pos": true,
+    "obs_pathing_grid": false,
+    "obs_starcraft": true,
+    "obs_terrain_height": false,
+    "obs_timestep_number": false,
+    "replay_dir": "",
+    "replay_prefix": "",
+    "reward_death_value": 10,
+    "reward_defeat": 0,
+    "reward_negative_scale": 0.5,
+    "reward_only_positive": true,
+    "reward_scale": true,
+    "reward_scale_rate": 20,
+    "reward_sparse": false,
+    "reward_win": 200,
+    "seed": null,
+    "state_last_action": true,
+    "state_timestep_number": false,
+    "step_mul": 8
+  },
+  "epsilon_anneal_time": 100000,
+  "epsilon_finish": 0.05,
+  "epsilon_start": 1.0,
+  "evaluate": false,
+  "gamma": 0.99,
+  "goal_dim": 16,
+  "grad_norm_clip": 10,
+  "hypernet_embed": 64,
+  "hypernet_layers": 2,
+  "intrinsic_rewards_alpha": 0.001,
+  "label": "default_label",
+  "learner": "feudal_learner",
+  "learner_log_interval": 10000,
+  "load_step": 0,
+  "local_results_path": "results",
+  "log_interval": 10000,
+  "lr": 0.0005,
+  "mac": "feudal_mac",
+  "manager_hidden_dim": 64,
+  "mixer": "qmix",
+  "mixing_embed_dim": 32,
+  "name": "feudal",
+  "obs_agent_id": true,
+  "obs_last_action": false,
+  "optim_alpha": 0.99,
+  "optim_eps": 1e-05,
+  "repeat_id": 1,
+  "rnn_hidden_dim": 64,
+  "run": "default",
+  "runner": "parallel",
+  "runner_log_interval": 10000,
+  "save_model": true,
+  "save_model_interval": 2000000,
+  "save_replay": false,
+  "seed": 414411342,
+  "state_dim": 80,
+  "t_max": 4050000,
+  "target_update_interval": 200,
+  "td_lambda": 0.6,
+  "test_greedy": true,
+  "test_interval": 10000,
+  "test_nepisode": 32,
+  "thread_num": 4,
+  "use_cuda": true,
+  "use_tensorboard": false,
+  "vf_coef": 0.5,
+  "worker_hidden_dim": 64
+}

results/sacred/10gen_protoss/feudal/2/info.json

@@ -0,0 +1,136 @@
+{
+  "battle_won_mean": [
+    0.0
+  ],
+  "battle_won_mean_T": [
+    265
+  ],
+  "dead_allies_mean": [
+    5.0
+  ],
+  "dead_allies_mean_T": [
+    265
+  ],
+  "dead_enemies_mean": [
+    1.25
+  ],
+  "dead_enemies_mean_T": [
+    265
+  ],
+  "ep_length_mean": [
+    66.25
+  ],
+  "ep_length_mean_T": [
+    265
+  ],
+  "epsilon": [
+    1.0
+  ],
+  "epsilon_T": [
+    265
+  ],
+  "return_max": [
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 9.983471074380164
+    }
+  ],
+  "return_max_T": [
+    265
+  ],
+  "return_mean": [
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 6.397180549212854
+    }
+  ],
+  "return_mean_T": [
+    265
+  ],
+  "return_min": [
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 2.0509708737864076
+    }
+  ],
+  "return_min_T": [
+    265
+  ],
+  "return_std": [
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 3.1585009546446443
+    }
+  ],
+  "return_std_T": [
+    265
+  ],
+  "test_battle_won_mean": [
+    0.0
+  ],
+  "test_battle_won_mean_T": [
+    265
+  ],
+  "test_dead_allies_mean": [
+    5.0
+  ],
+  "test_dead_allies_mean_T": [
+    265
+  ],
+  "test_dead_enemies_mean": [
+    0.125
+  ],
+  "test_dead_enemies_mean_T": [
+    265
+  ],
+  "test_ep_length_mean": [
+    56.84375
+  ],
+  "test_ep_length_mean_T": [
+    265
+  ],
+  "test_return_max": [
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 5.96078431372549
+    }
+  ],
+  "test_return_max_T": [
+    265
+  ],
+  "test_return_mean": [
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 1.1870757990039804
+    }
+  ],
+  "test_return_mean_T": [
+    265
+  ],
+  "test_return_min": [
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 0.0
+    }
+  ],
+  "test_return_min_T": [
+    265
+  ],
+  "test_return_std": [
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 1.5675980789607145
+    }
+  ],
+  "test_return_std_T": [
+    265
+  ]
+}

results/sacred/10gen_protoss/feudal/2/metrics.json

@@ -0,0 +1 @@
+{}

results/sacred/10gen_protoss/feudal/2/run.json

@@ -0,0 +1,125 @@
+{
+  "artifacts": [],
+  "command": "my_main",
+  "experiment": {
+    "base_dir": "C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src",
+    "dependencies": [
+      "numpy==1.23.1",
+      "PyYAML==6.0.2",
+      "sacred==0.8.7",
+      "torch==1.13.1+cu117"
+    ],
+    "mainfile": "main.py",
+    "name": "pymarl",
+    "repositories": [
+      {
+        "commit": "44bb188185fd0292a1a306c86396027efb44224d",
+        "dirty": true,
+        "url": "https://github.com/tjuHaoXiaotian/pymarl3.git"
+      },
+      {
+        "commit": "44bb188185fd0292a1a306c86396027efb44224d",
+        "dirty": true,
+        "url": "https://github.com/tjuHaoXiaotian/pymarl3.git"
+      }
+    ],
+    "sources": [
+      [
+        "main.py",
+        "_sources\\main_654daaa6534bcee62784d639ea63e51d.py"
+      ],
+      [
+        "utils\\logging.py",
+        "_sources\\logging_f71df6d788e929fac28afdf951d63d54.py"
+      ]
+    ]
+  },
+  "fail_trace": [
+    "Traceback (most recent call last):\n",
+    "  File \"C:\\Users\\Taiyo\\.conda\\envs\\SMACV2\\lib\\site-packages\\sacred\\config\\captured_function.py\", line 42, in captured_function\n    result = wrapped(*args, **kwargs)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\main.py\", line 44, in my_main\n    run_REGISTRY[_config['run']](_run, config, _log)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\run\\run.py\", line 128, in run\n    run_sequential(args=args, logger=logger)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\run\\run.py\", line 280, in run_sequential\n    learner.train(episode_sample, runner.t_env, episode)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\learners\\FeUdal_learner.py\", line 148, in train\n    cos_sim = F.cosine_similarity(temp_state_diff, temp_goals, dim=-1)\n",
+    "RuntimeError: The size of tensor a (64768) must match the size of tensor b (56320) at non-singleton dimension 0\n"
+  ],
+  "heartbeat": "2025-01-05T19:50:34.270485",
+  "host": {
+    "ENV": {},
+    "cpu": "AMD Ryzen 7 5700X3D 8-Core Processor",
+    "gpus": {
+      "driver_version": "560.94",
+      "gpus": [
+        {
+          "model": "NVIDIA GeForce RTX 4080 SUPER",
+          "persistence_mode": false,
+          "total_memory": 16376
+        }
+      ]
+    },
+    "hostname": "Taiyopen",
+    "os": [
+      "Windows",
+      "Windows-10-10.0.22631-SP0"
+    ],
+    "python_version": "3.10.16"
+  },
+  "meta": {
+    "command": "my_main",
+    "config_updates": {
+      "batch_size": 128,
+      "batch_size_run": 4,
+      "buffer_size": 5000,
+      "epsilon_anneal_time": 100000,
+      "obs_agent_id": true,
+      "obs_last_action": false,
+      "runner": "parallel",
+      "t_max": 4050000,
+      "td_lambda": 0.6,
+      "use_tensorboard": false
+    },
+    "named_configs": [],
+    "options": {
+      "--beat-interval": null,
+      "--capture": null,
+      "--comment": null,
+      "--debug": false,
+      "--enforce_clean": false,
+      "--file_storage": null,
+      "--force": false,
+      "--help": false,
+      "--id": null,
+      "--loglevel": null,
+      "--mongo_db": null,
+      "--name": null,
+      "--pdb": false,
+      "--print-config": false,
+      "--priority": null,
+      "--queue": false,
+      "--s3": null,
+      "--sql": null,
+      "--tiny_db": null,
+      "--unobserved": false,
+      "COMMAND": null,
+      "UPDATE": [
+        "obs_agent_id=True",
+        "obs_last_action=False",
+        "runner=parallel",
+        "batch_size_run=4",
+        "buffer_size=5000",
+        "t_max=4050000",
+        "epsilon_anneal_time=100000",
+        "batch_size=128",
+        "td_lambda=0.6",
+        "use_tensorboard=False"
+      ],
+      "help": false,
+      "with": true
+    }
+  },
+  "resources": [],
+  "result": null,
+  "start_time": "2025-01-05T19:49:13.745305",
+  "status": "FAILED",
+  "stop_time": "2025-01-05T19:50:34.274484"
+}

results/sacred/10gen_protoss/feudal/3/config.json

@@ -0,0 +1,131 @@
+{
+  "action_selector": "epsilon_greedy",
+  "agent": "feudal",
+  "agent_output_type": "q",
+  "asn_hidden_size": 32,
+  "batch_size": 128,
+  "batch_size_run": 4,
+  "buffer_cpu_only": true,
+  "buffer_size": 5000,
+  "c": 1,
+  "checkpoint_path": "",
+  "cpu_inference": true,
+  "critic_hidden_dim": 64,
+  "critic_lr": 0.0005,
+  "double_q": true,
+  "embedding_dim": 16,
+  "enable_parallel_computing": false,
+  "env": "sc2_v2",
+  "env_args": {
+    "capability_config": {
+      "n_units": 5,
+      "start_positions": {
+        "dist_type": "surrounded_and_reflect",
+        "map_x": 32,
+        "map_y": 32,
+        "n_enemies": 5,
+        "p": 0.5
+      },
+      "team_gen": {
+        "dist_type": "weighted_teams",
+        "exception_unit_types": [
+          "colossus"
+        ],
+        "observe": true,
+        "unit_types": [
+          "stalker",
+          "zealot",
+          "colossus"
+        ],
+        "weights": [
+          0.45,
+          0.45,
+          0.1
+        ]
+      }
+    },
+    "change_fov_with_move": false,
+    "conic_fov": false,
+    "continuing_episode": false,
+    "debug": false,
+    "difficulty": "7",
+    "fully_observable": false,
+    "game_version": null,
+    "heuristic_ai": false,
+    "heuristic_rest": false,
+    "kill_unit_step_mul": 2,
+    "map_name": "10gen_protoss",
+    "move_amount": 2,
+    "num_fov_actions": 12,
+    "obs_all_health": true,
+    "obs_instead_of_state": false,
+    "obs_last_action": false,
+    "obs_own_health": true,
+    "obs_own_pos": true,
+    "obs_pathing_grid": false,
+    "obs_starcraft": true,
+    "obs_terrain_height": false,
+    "obs_timestep_number": false,
+    "replay_dir": "",
+    "replay_prefix": "",
+    "reward_death_value": 10,
+    "reward_defeat": 0,
+    "reward_negative_scale": 0.5,
+    "reward_only_positive": true,
+    "reward_scale": true,
+    "reward_scale_rate": 20,
+    "reward_sparse": false,
+    "reward_win": 200,
+    "seed": null,
+    "state_last_action": true,
+    "state_timestep_number": false,
+    "step_mul": 8
+  },
+  "epsilon_anneal_time": 100000,
+  "epsilon_finish": 0.05,
+  "epsilon_start": 1.0,
+  "evaluate": false,
+  "gamma": 0.99,
+  "goal_dim": 16,
+  "grad_norm_clip": 10,
+  "hypernet_embed": 64,
+  "hypernet_layers": 2,
+  "intrinsic_rewards_alpha": 0.001,
+  "label": "default_label",
+  "learner": "feudal_learner",
+  "learner_log_interval": 10000,
+  "load_step": 0,
+  "local_results_path": "results",
+  "log_interval": 10000,
+  "lr": 0.0005,
+  "mac": "feudal_mac",
+  "manager_hidden_dim": 64,
+  "mixer": "qmix",
+  "mixing_embed_dim": 32,
+  "name": "feudal",
+  "obs_agent_id": true,
+  "obs_last_action": false,
+  "optim_alpha": 0.99,
+  "optim_eps": 1e-05,
+  "repeat_id": 1,
+  "rnn_hidden_dim": 64,
+  "run": "default",
+  "runner": "parallel",
+  "runner_log_interval": 10000,
+  "save_model": true,
+  "save_model_interval": 2000000,
+  "save_replay": false,
+  "seed": 521291150,
+  "state_dim": 92,
+  "t_max": 4050000,
+  "target_update_interval": 200,
+  "td_lambda": 0.6,
+  "test_greedy": true,
+  "test_interval": 10000,
+  "test_nepisode": 32,
+  "thread_num": 4,
+  "use_cuda": true,
+  "use_tensorboard": false,
+  "vf_coef": 0.5,
+  "worker_hidden_dim": 64
+}

results/sacred/10gen_protoss/feudal/3/info.json

@@ -0,0 +1,368 @@
+{
+  "battle_won_mean": [
+    0.0,
+    0.0
+  ],
+  "battle_won_mean_T": [
+    211,
+    10252
+  ],
+  "dead_allies_mean": [
+    5.0,
+    5.0
+  ],
+  "dead_allies_mean_T": [
+    211,
+    10252
+  ],
+  "dead_enemies_mean": [
+    0.5,
+    0.3111111111111111
+  ],
+  "dead_enemies_mean_T": [
+    211,
+    10252
+  ],
+  "ep_length_mean": [
+    52.75,
+    55.78333333333333
+  ],
+  "ep_length_mean_T": [
+    211,
+    10252
+  ],
+  "episode": [
+    180
+  ],
+  "episode_T": [
+    10030
+  ],
+  "episode_in_buffer": [
+    180
+  ],
+  "episode_in_buffer_T": [
+    10030
+  ],
+  "epsilon": [
+    1.0,
+    0.904715
+  ],
+  "epsilon_T": [
+    211,
+    10252
+  ],
+  "grad_norm_manager": [
+    {
+      "py/reduce": [
+        {
+          "py/function": "torch._utils._rebuild_tensor_v2"
+        },
+        {
+          "py/tuple": [
+            {
+              "py/reduce": [
+                {
+                  "py/function": "torch.storage._load_from_bytes"
+                },
+                {
+                  "py/tuple": [
+                    {
+                      "py/b64": "gAKKCmz8nEb5IGqoUBkugAJN6QMugAJ9cQAoWBAAAABwcm90b2NvbF92ZXJzaW9ucQFN6QNYDQAAAGxpdHRsZV9lbmRpYW5xAohYCgAAAHR5cGVfc2l6ZXNxA31xBChYBQAAAHNob3J0cQVLAlgDAAAAaW50cQZLBFgEAAAAbG9uZ3EHSwR1dS6AAihYBwAAAHN0b3JhZ2VxAGN0b3JjaApGbG9hdFN0b3JhZ2UKcQFYDQAAADIxNDAzMDE2ODY1NjBxAlgGAAAAY3VkYTowcQNLAU50cQRRLoACXXEAWA0AAAAyMTQwMzAxNjg2NTYwcQFhLgEAAAAAAAAAnTV0RA=="
+                    }
+                  ]
+                }
+              ]
+            },
+            0,
+            {
+              "py/tuple": []
+            },
+            {
+              "py/tuple": []
+            },
+            false,
+            {
+              "py/reduce": [
+                {
+                  "py/type": "collections.OrderedDict"
+                },
+                {
+                  "py/tuple": []
+                },
+                null,
+                null,
+                {
+                  "py/tuple": []
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    }
+  ],
+  "grad_norm_manager_T": [
+    7135
+  ],
+  "grad_norm_worker": [
+    {
+      "py/reduce": [
+        {
+          "py/function": "torch._utils._rebuild_tensor_v2"
+        },
+        {
+          "py/tuple": [
+            {
+              "py/reduce": [
+                {
+                  "py/function": "torch.storage._load_from_bytes"
+                },
+                {
+                  "py/tuple": [
+                    {
+                      "py/b64": "gAKKCmz8nEb5IGqoUBkugAJN6QMugAJ9cQAoWBAAAABwcm90b2NvbF92ZXJzaW9ucQFN6QNYDQAAAGxpdHRsZV9lbmRpYW5xAohYCgAAAHR5cGVfc2l6ZXNxA31xBChYBQAAAHNob3J0cQVLAlgDAAAAaW50cQZLBFgEAAAAbG9uZ3EHSwR1dS6AAihYBwAAAHN0b3JhZ2VxAGN0b3JjaApGbG9hdFN0b3JhZ2UKcQFYDQAAADIxNDAzMDE2NjI2NTZxAlgGAAAAY3VkYTowcQNLAU50cQRRLoACXXEAWA0AAAAyMTQwMzAxNjYyNjU2cQFhLgEAAAAAAAAAHtgcRg=="
+                    }
+                  ]
+                }
+              ]
+            },
+            0,
+            {
+              "py/tuple": []
+            },
+            {
+              "py/tuple": []
+            },
+            false,
+            {
+              "py/reduce": [
+                {
+                  "py/type": "collections.OrderedDict"
+                },
+                {
+                  "py/tuple": []
+                },
+                null,
+                null,
+                {
+                  "py/tuple": []
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    }
+  ],
+  "grad_norm_worker_T": [
+    7135
+  ],
+  "hit_prob": [
+    0.4580238461494446
+  ],
+  "hit_prob_T": [
+    7135
+  ],
+  "manager_advantage": [
+    0.1406308114528656
+  ],
+  "manager_advantage_T": [
+    7135
+  ],
+  "manager_cos_sim": [
+    -0.00032846396788954735
+  ],
+  "manager_cos_sim_T": [
+    7135
+  ],
+  "manager_loss": [
+    10.693564414978027
+  ],
+  "manager_loss_T": [
+    7135
+  ],
+  "q_taken_mean": [
+    0.06587212437587596
+  ],
+  "q_taken_mean_T": [
+    7135
+  ],
+  "return_max": [
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 7.432926829268293
+    },
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 14.95343137254902
+    }
+  ],
+  "return_max_T": [
+    211,
+    10252
+  ],
+  "return_mean": [
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 4.3768681252989
+    },
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 4.579137772995377
+    }
+  ],
+  "return_mean_T": [
+    211,
+    10252
+  ],
+  "return_min": [
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 2.2794117647058827
+    },
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 1.4308943089430894
+    }
+  ],
+  "return_min_T": [
+    211,
+    10252
+  ],
+  "return_std": [
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 1.89094258138571
+    },
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 2.421259942116347
+    }
+  ],
+  "return_std_T": [
+    211,
+    10252
+  ],
+  "target_mean": [
+    0.10347526224159075
+  ],
+  "target_mean_T": [
+    7135
+  ],
+  "td_error_abs": [
+    0.22424918631197444
+  ],
+  "td_error_abs_T": [
+    7135
+  ],
+  "test_battle_won_mean": [
+    0.0,
+    0.0
+  ],
+  "test_battle_won_mean_T": [
+    211,
+    10252
+  ],
+  "test_dead_allies_mean": [
+    5.0,
+    5.0
+  ],
+  "test_dead_allies_mean_T": [
+    211,
+    10252
+  ],
+  "test_dead_enemies_mean": [
+    0.34375,
+    1.0
+  ],
+  "test_dead_enemies_mean_T": [
+    211,
+    10252
+  ],
+  "test_ep_length_mean": [
+    52.21875,
+    57.5625
+  ],
+  "test_ep_length_mean_T": [
+    211,
+    10252
+  ],
+  "test_return_max": [
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 6.959349593495936
+    },
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 19.607843137254893
+    }
+  ],
+  "test_return_max_T": [
+    211,
+    10252
+  ],
+  "test_return_mean": [
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 2.185841104734577
+    },
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 4.841020668938307
+    }
+  ],
+  "test_return_mean_T": [
+    211,
+    10252
+  ],
+  "test_return_min": [
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 0.0
+    },
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 0.0
+    }
+  ],
+  "test_return_min_T": [
+    211,
+    10252
+  ],
+  "test_return_std": [
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 2.2407881654472965
+    },
+    {
+      "dtype": "float64",
+      "py/object": "numpy.float64",
+      "value": 4.010960846314079
+    }
+  ],
+  "test_return_std_T": [
+    211,
+    10252
+  ],
+  "worker_loss": [
+    714.7169189453125
+  ],
+  "worker_loss_T": [
+    7135
+  ]
+}

results/sacred/10gen_protoss/feudal/3/metrics.json

@@ -0,0 +1 @@
+{}

results/sacred/10gen_protoss/feudal/3/run.json

@@ -0,0 +1,116 @@
+{
+  "artifacts": [],
+  "command": "my_main",
+  "experiment": {
+    "base_dir": "C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src",
+    "dependencies": [
+      "numpy==1.23.1",
+      "PyYAML==6.0.2",
+      "sacred==0.8.7",
+      "torch==1.13.1+cu117"
+    ],
+    "mainfile": "main.py",
+    "name": "pymarl",
+    "repositories": [
+      {
+        "commit": "44bb188185fd0292a1a306c86396027efb44224d",
+        "dirty": true,
+        "url": "https://github.com/tjuHaoXiaotian/pymarl3.git"
+      },
+      {
+        "commit": "44bb188185fd0292a1a306c86396027efb44224d",
+        "dirty": true,
+        "url": "https://github.com/tjuHaoXiaotian/pymarl3.git"
+      }
+    ],
+    "sources": [
+      [
+        "main.py",
+        "_sources\\main_654daaa6534bcee62784d639ea63e51d.py"
+      ],
+      [
+        "utils\\logging.py",
+        "_sources\\logging_f71df6d788e929fac28afdf951d63d54.py"
+      ]
+    ]
+  },
+  "heartbeat": "2025-01-05T19:53:52.751678",
+  "host": {
+    "ENV": {},
+    "cpu": "AMD Ryzen 7 5700X3D 8-Core Processor",
+    "gpus": {
+      "driver_version": "560.94",
+      "gpus": [
+        {
+          "model": "NVIDIA GeForce RTX 4080 SUPER",
+          "persistence_mode": false,
+          "total_memory": 16376
+        }
+      ]
+    },
+    "hostname": "Taiyopen",
+    "os": [
+      "Windows",
+      "Windows-10-10.0.22631-SP0"
+    ],
+    "python_version": "3.10.16"
+  },
+  "meta": {
+    "command": "my_main",
+    "config_updates": {
+      "batch_size": 128,
+      "batch_size_run": 4,
+      "buffer_size": 5000,
+      "epsilon_anneal_time": 100000,
+      "obs_agent_id": true,
+      "obs_last_action": false,
+      "runner": "parallel",
+      "t_max": 4050000,
+      "td_lambda": 0.6,
+      "use_tensorboard": false
+    },
+    "named_configs": [],
+    "options": {
+      "--beat-interval": null,
+      "--capture": null,
+      "--comment": null,
+      "--debug": false,
+      "--enforce_clean": false,
+      "--file_storage": null,
+      "--force": false,
+      "--help": false,
+      "--id": null,
+      "--loglevel": null,
+      "--mongo_db": null,
+      "--name": null,
+      "--pdb": false,
+      "--print-config": false,
+      "--priority": null,
+      "--queue": false,
+      "--s3": null,
+      "--sql": null,
+      "--tiny_db": null,
+      "--unobserved": false,
+      "COMMAND": null,
+      "UPDATE": [
+        "obs_agent_id=True",
+        "obs_last_action=False",
+        "runner=parallel",
+        "batch_size_run=4",
+        "buffer_size=5000",
+        "t_max=4050000",
+        "epsilon_anneal_time=100000",
+        "batch_size=128",
+        "td_lambda=0.6",
+        "use_tensorboard=False"
+      ],
+      "help": false,
+      "with": true
+    }
+  },
+  "resources": [],
+  "result": null,
+  "start_time": "2025-01-05T19:51:28.620969",
+  "status": "INTERRUPTED",
+  "stop_time": "2025-01-05T19:53:52.757749"
+}

results/sacred/10gen_protoss/feudal/4/config.json

@@ -0,0 +1,131 @@
+{
+  "action_selector": "epsilon_greedy",
+  "agent": "feudal",
+  "agent_output_type": "q",
+  "asn_hidden_size": 32,
+  "batch_size": 128,
+  "batch_size_run": 4,
+  "buffer_cpu_only": true,
+  "buffer_size": 5000,
+  "c": 1,
+  "checkpoint_path": "",
+  "cpu_inference": true,
+  "critic_hidden_dim": 64,
+  "critic_lr": 0.0005,
+  "double_q": true,
+  "embedding_dim": 16,
+  "enable_parallel_computing": false,
+  "env": "sc2_v2",
+  "env_args": {
+    "capability_config": {
+      "n_units": 5,
+      "start_positions": {
+        "dist_type": "surrounded_and_reflect",
+        "map_x": 32,
+        "map_y": 32,
+        "n_enemies": 5,
+        "p": 0.5
+      },
+      "team_gen": {
+        "dist_type": "weighted_teams",
+        "exception_unit_types": [
+          "colossus"
+        ],
+        "observe": true,
+        "unit_types": [
+          "stalker",
+          "zealot",
+          "colossus"
+        ],
+        "weights": [
+          0.45,
+          0.45,
+          0.1
+        ]
+      }
+    },
+    "change_fov_with_move": false,
+    "conic_fov": false,
+    "continuing_episode": false,
+    "debug": false,
+    "difficulty": "7",
+    "fully_observable": false,
+    "game_version": null,
+    "heuristic_ai": false,
+    "heuristic_rest": false,
+    "kill_unit_step_mul": 2,
+    "map_name": "10gen_protoss",
+    "move_amount": 2,
+    "num_fov_actions": 12,
+    "obs_all_health": true,
+    "obs_instead_of_state": false,
+    "obs_last_action": false,
+    "obs_own_health": true,
+    "obs_own_pos": true,
+    "obs_pathing_grid": false,
+    "obs_starcraft": true,
+    "obs_terrain_height": false,
+    "obs_timestep_number": false,
+    "replay_dir": "",
+    "replay_prefix": "",
+    "reward_death_value": 10,
+    "reward_defeat": 0,
+    "reward_negative_scale": 0.5,
+    "reward_only_positive": true,
+    "reward_scale": true,
+    "reward_scale_rate": 20,
+    "reward_sparse": false,
+    "reward_win": 200,
+    "seed": null,
+    "state_last_action": true,
+    "state_timestep_number": false,
+    "step_mul": 8
+  },
+  "epsilon_anneal_time": 100000,
+  "epsilon_finish": 0.05,
+  "epsilon_start": 1.0,
+  "evaluate": false,
+  "gamma": 0.99,
+  "goal_dim": 16,
+  "grad_norm_clip": 10,
+  "hypernet_embed": 64,
+  "hypernet_layers": 2,
+  "intrinsic_rewards_alpha": 0.001,
+  "label": "default_label",
+  "learner": "feudal_learner",
+  "learner_log_interval": 10000,
+  "load_step": 0,
+  "local_results_path": "results",
+  "log_interval": 10000,
+  "lr": 0.0005,
+  "mac": "feudal_mac",
+  "manager_hidden_dim": 64,
+  "mixer": "qmix",
+  "mixing_embed_dim": 32,
+  "name": "feudal",
+  "obs_agent_id": true,
+  "obs_last_action": false,
+  "optim_alpha": 0.99,
+  "optim_eps": 1e-05,
+  "repeat_id": 1,
+  "rnn_hidden_dim": 64,
+  "run": "default",
+  "runner": "parallel",
+  "runner_log_interval": 10000,
+  "save_model": true,
+  "save_model_interval": 2000000,
+  "save_replay": false,
+  "seed": 346146537,
+  "state_dim": 92,
+  "t_max": 4050000,
+  "target_update_interval": 200,
+  "td_lambda": 0.6,
+  "test_greedy": true,
+  "test_interval": 10000,
+  "test_nepisode": 32,
+  "thread_num": 4,
+  "use_cuda": true,
+  "use_tensorboard": false,
+  "vf_coef": 0.5,
+  "worker_hidden_dim": 64
+}

results/sacred/10gen_protoss/feudal/4/metrics.json

@@ -0,0 +1 @@
+{}

results/sacred/10gen_protoss/feudal/4/run.json

@@ -0,0 +1,116 @@
+{
+  "artifacts": [],
+  "command": "my_main",
+  "experiment": {
+    "base_dir": "C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src",
+    "dependencies": [
+      "numpy==1.23.1",
+      "PyYAML==6.0.2",
+      "sacred==0.8.7",
+      "torch==1.13.1+cu117"
+    ],
+    "mainfile": "main.py",
+    "name": "pymarl",
+    "repositories": [
+      {
+        "commit": "44bb188185fd0292a1a306c86396027efb44224d",
+        "dirty": true,
+        "url": "https://github.com/tjuHaoXiaotian/pymarl3.git"
+      },
+      {
+        "commit": "44bb188185fd0292a1a306c86396027efb44224d",
+        "dirty": true,
+        "url": "https://github.com/tjuHaoXiaotian/pymarl3.git"
+      }
+    ],
+    "sources": [
+      [
+        "main.py",
+        "_sources\\main_654daaa6534bcee62784d639ea63e51d.py"
+      ],
+      [
+        "utils\\logging.py",
+        "_sources\\logging_f71df6d788e929fac28afdf951d63d54.py"
+      ]
+    ]
+  },
+  "heartbeat": "2025-01-05T20:17:27.881164",
+  "host": {
+    "ENV": {},
+    "cpu": "AMD Ryzen 7 5700X3D 8-Core Processor",
+    "gpus": {
+      "driver_version": "560.94",
+      "gpus": [
+        {
+          "model": "NVIDIA GeForce RTX 4080 SUPER",
+          "persistence_mode": false,
+          "total_memory": 16376
+        }
+      ]
+    },
+    "hostname": "Taiyopen",
+    "os": [
+      "Windows",
+      "Windows-10-10.0.22631-SP0"
+    ],
+    "python_version": "3.10.16"
+  },
+  "meta": {
+    "command": "my_main",
+    "config_updates": {
+      "batch_size": 128,
+      "batch_size_run": 4,
+      "buffer_size": 5000,
+      "epsilon_anneal_time": 100000,
+      "obs_agent_id": true,
+      "obs_last_action": false,
+      "runner": "parallel",
+      "t_max": 4050000,
+      "td_lambda": 0.6,
+      "use_tensorboard": false
+    },
+    "named_configs": [],
+    "options": {
+      "--beat-interval": null,
+      "--capture": null,
+      "--comment": null,
+      "--debug": false,
+      "--enforce_clean": false,
+      "--file_storage": null,
+      "--force": false,
+      "--help": false,
+      "--id": null,
+      "--loglevel": null,
+      "--mongo_db": null,
+      "--name": null,
+      "--pdb": false,
+      "--print-config": false,
+      "--priority": null,
+      "--queue": false,
+      "--s3": null,
+      "--sql": null,
+      "--tiny_db": null,
+      "--unobserved": false,
+      "COMMAND": null,
+      "UPDATE": [
+        "obs_agent_id=True",
+        "obs_last_action=False",
+        "runner=parallel",
+        "batch_size_run=4",
+        "buffer_size=5000",
+        "t_max=4050000",
+        "epsilon_anneal_time=100000",
+        "batch_size=128",
+        "td_lambda=0.6",
+        "use_tensorboard=False"
+      ],
+      "help": false,
+      "with": true
+    }
+  },
+  "resources": [],
+  "result": null,
+  "start_time": "2025-01-05T19:55:31.376680",
+  "status": "INTERRUPTED",
+  "stop_time": "2025-01-05T20:17:27.914784"
+}

results/sacred/10gen_protoss/feudal/5/config.json

@@ -0,0 +1,131 @@
+{
+  "action_selector": "epsilon_greedy",
+  "agent": "feudal",
+  "agent_output_type": "q",
+  "asn_hidden_size": 32,
+  "batch_size": 128,
+  "batch_size_run": 4,
+  "buffer_cpu_only": true,
+  "buffer_size": 5000,
+  "c": 1,
+  "checkpoint_path": "",
+  "cpu_inference": true,
+  "critic_hidden_dim": 64,
+  "critic_lr": 0.0005,
+  "double_q": true,
+  "embedding_dim": 16,
+  "enable_parallel_computing": false,
+  "env": "sc2_v2",
+  "env_args": {
+    "capability_config": {
+      "n_units": 5,
+      "start_positions": {
+        "dist_type": "surrounded_and_reflect",
+        "map_x": 32,
+        "map_y": 32,
+        "n_enemies": 5,
+        "p": 0.5
+      },
+      "team_gen": {
+        "dist_type": "weighted_teams",
+        "exception_unit_types": [
+          "colossus"
+        ],
+        "observe": true,
+        "unit_types": [
+          "stalker",
+          "zealot",
+          "colossus"
+        ],
+        "weights": [
+          0.45,
+          0.45,
+          0.1
+        ]
+      }
+    },
+    "change_fov_with_move": false,
+    "conic_fov": false,
+    "continuing_episode": false,
+    "debug": false,
+    "difficulty": "7",
+    "fully_observable": false,
+    "game_version": null,
+    "heuristic_ai": false,
+    "heuristic_rest": false,
+    "kill_unit_step_mul": 2,
+    "map_name": "10gen_protoss",
+    "move_amount": 2,
+    "num_fov_actions": 12,
+    "obs_all_health": true,
+    "obs_instead_of_state": false,
+    "obs_last_action": false,
+    "obs_own_health": true,
+    "obs_own_pos": true,
+    "obs_pathing_grid": false,
+    "obs_starcraft": true,
+    "obs_terrain_height": false,
+    "obs_timestep_number": false,
+    "replay_dir": "",
+    "replay_prefix": "",
+    "reward_death_value": 10,
+    "reward_defeat": 0,
+    "reward_negative_scale": 0.5,
+    "reward_only_positive": true,
+    "reward_scale": true,
+    "reward_scale_rate": 20,
+    "reward_sparse": false,
+    "reward_win": 200,
+    "seed": null,
+    "state_last_action": true,
+    "state_timestep_number": false,
+    "step_mul": 8
+  },
+  "epsilon_anneal_time": 100000,
+  "epsilon_finish": 0.05,
+  "epsilon_start": 1.0,
+  "evaluate": false,
+  "gamma": 0.99,
+  "goal_dim": 16,
+  "grad_norm_clip": 10,
+  "hypernet_embed": 64,
+  "hypernet_layers": 2,
+  "intrinsic_rewards_alpha": 0.001,
+  "label": "default_label",
+  "learner": "feudal_learner",
+  "learner_log_interval": 10000,
+  "load_step": 0,
+  "local_results_path": "results",
+  "log_interval": 10000,
+  "lr": 0.0005,
+  "mac": "feudal_mac",
+  "manager_hidden_dim": 64,
+  "mixer": "qmix",
+  "mixing_embed_dim": 32,
+  "name": "feudal",
+  "obs_agent_id": true,
+  "obs_last_action": false,
+  "optim_alpha": 0.99,
+  "optim_eps": 1e-05,
+  "repeat_id": 1,
+  "rnn_hidden_dim": 64,
+  "run": "default",
+  "runner": "parallel",
+  "runner_log_interval": 10000,
+  "save_model": true,
+  "save_model_interval": 2000000,
+  "save_replay": false,
+  "seed": 549441411,
+  "state_dim": 92,
+  "t_max": 4050000,
+  "target_update_interval": 200,
+  "td_lambda": 0.6,
+  "test_greedy": true,
+  "test_interval": 10000,
+  "test_nepisode": 32,
+  "thread_num": 4,
+  "use_cuda": true,
+  "use_tensorboard": false,
+  "vf_coef": 0.5,
+  "worker_hidden_dim": 64
+}

results/sacred/10gen_protoss/feudal/5/metrics.json

@@ -0,0 +1 @@
+{}

results/sacred/10gen_protoss/feudal/5/run.json

@@ -0,0 +1,124 @@
+{
+  "artifacts": [],
+  "command": "my_main",
+  "experiment": {
+    "base_dir": "C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src",
+    "dependencies": [
+      "numpy==1.23.1",
+      "PyYAML==6.0.2",
+      "sacred==0.8.7",
+      "torch==1.13.1+cu117"
+    ],
+    "mainfile": "main.py",
+    "name": "pymarl",
+    "repositories": [
+      {
+        "commit": "44bb188185fd0292a1a306c86396027efb44224d",
+        "dirty": true,
+        "url": "https://github.com/tjuHaoXiaotian/pymarl3.git"
+      },
+      {
+        "commit": "44bb188185fd0292a1a306c86396027efb44224d",
+        "dirty": true,
+        "url": "https://github.com/tjuHaoXiaotian/pymarl3.git"
+      }
+    ],
+    "sources": [
+      [
+        "main.py",
+        "_sources\\main_654daaa6534bcee62784d639ea63e51d.py"
+      ],
+      [
+        "utils\\logging.py",
+        "_sources\\logging_f71df6d788e929fac28afdf951d63d54.py"
+      ]
+    ]
+  },
+  "fail_trace": [
+    "Traceback (most recent call last):\n",
+    "  File \"C:\\Users\\Taiyo\\.conda\\envs\\SMACV2\\lib\\site-packages\\sacred\\config\\captured_function.py\", line 42, in captured_function\n    result = wrapped(*args, **kwargs)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\main.py\", line 44, in my_main\n    run_REGISTRY[_config['run']](_run, config, _log)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\run\\run.py\", line 128, in run\n    run_sequential(args=args, logger=logger)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\run\\run.py\", line 318, in run_sequential\n    sys.stdout.flush()\n",
+    "NameError: name 'sys' is not defined\n"
+  ],
+  "heartbeat": "2025-01-06T04:05:20.393530",
+  "host": {
+    "ENV": {},
+    "cpu": "AMD Ryzen 7 5700X3D 8-Core Processor",
+    "gpus": {
+      "driver_version": "560.94",
+      "gpus": [
+        {
+          "model": "NVIDIA GeForce RTX 4080 SUPER",
+          "persistence_mode": false,
+          "total_memory": 16376
+        }
+      ]
+    },
+    "hostname": "Taiyopen",
+    "os": [
+      "Windows",
+      "Windows-10-10.0.22631-SP0"
+    ],
+    "python_version": "3.10.16"
+  },
+  "meta": {
+    "command": "my_main",
+    "config_updates": {
+      "batch_size": 128,
+      "batch_size_run": 4,
+      "buffer_size": 5000,
+      "epsilon_anneal_time": 100000,
+      "obs_agent_id": true,
+      "obs_last_action": false,
+      "runner": "parallel",
+      "t_max": 4050000,
+      "td_lambda": 0.6,
+      "use_tensorboard": false
+    },
+    "named_configs": [],
+    "options": {
+      "--beat-interval": null,
+      "--capture": null,
+      "--comment": null,
+      "--debug": false,
+      "--enforce_clean": false,
+      "--file_storage": null,
+      "--force": false,
+      "--help": false,
+      "--id": null,
+      "--loglevel": null,
+      "--mongo_db": null,
+      "--name": null,
+      "--pdb": false,
+      "--print-config": false,
+      "--priority": null,
+      "--queue": false,
+      "--s3": null,
+      "--sql": null,
+      "--tiny_db": null,
+      "--unobserved": false,
+      "COMMAND": null,
+      "UPDATE": [
+        "obs_agent_id=True",
+        "obs_last_action=False",
+        "runner=parallel",
+        "batch_size_run=4",
+        "buffer_size=5000",
+        "t_max=4050000",
+        "epsilon_anneal_time=100000",
+        "batch_size=128",
+        "td_lambda=0.6",
+        "use_tensorboard=False"
+      ],
+      "help": false,
+      "with": true
+    }
+  },
+  "resources": [],
+  "result": null,
+  "start_time": "2025-01-05T20:17:39.546682",
+  "status": "FAILED",
+  "stop_time": "2025-01-06T04:05:21.070681"
+}

results/sacred/10gen_protoss/feudal/_sources/logging_f71df6d788e929fac28afdf951d63d54.py

@@ -0,0 +1,68 @@
+from collections import defaultdict
+import logging
+import numpy as np
+import torch as th
+
+class Logger:
+    def __init__(self, console_logger):
+        self.console_logger = console_logger
+
+        self.use_tb = False
+        self.use_sacred = False
+        self.use_hdf = False
+
+        self.stats = defaultdict(lambda: [])
+
+    def setup_tb(self, directory_name):
+        # Import here so it doesn't have to be installed if you don't use it
+        from tensorboard_logger import configure, log_value
+        configure(directory_name)
+        self.tb_logger = log_value
+        self.use_tb = True
+
+    def setup_sacred(self, sacred_run_dict):
+        self.sacred_info = sacred_run_dict.info
+        self.use_sacred = True
+
+    def log_stat(self, key, value, t, to_sacred=True):
+        self.stats[key].append((t, value))
+
+        if self.use_tb:
+            self.tb_logger(key, value, t)
+
+        if self.use_sacred and to_sacred:
+            if key in self.sacred_info:
+                self.sacred_info["{}_T".format(key)].append(t)
+                self.sacred_info[key].append(value)
+            else:
+                self.sacred_info["{}_T".format(key)] = [t]
+                self.sacred_info[key] = [value]
+
+    def print_recent_stats(self):
+        log_str = "Recent Stats | t_env: {:>10} | Episode: {:>8}\n".format(*self.stats["episode"][-1])
+        i = 0
+        for (k, v) in sorted(self.stats.items()):
+            if k == "episode":
+                continue
+            i += 1
+            window = 5 if k != "epsilon" else 1
+            item = "{:.4f}".format(th.mean(th.tensor([float(x[1]) for x in self.stats[k][-window:]])))
+            log_str += "{:<25}{:>8}".format(k + ":", item)
+            log_str += "\n" if i % 4 == 0 else "\t"
+        self.console_logger.info(log_str)
+        # Reset stats to avoid accumulating logs in memory
+        self.stats = defaultdict(lambda: [])
+
+
+# set up a custom logger
+def get_logger():
+    logger = logging.getLogger()
+    logger.handlers = []
+    ch = logging.StreamHandler()
+    formatter = logging.Formatter('[%(levelname)s %(asctime)s] %(name)s %(message)s', '%H:%M:%S')
+    ch.setFormatter(formatter)
+    logger.addHandler(ch)
+    logger.setLevel('DEBUG')
+
+    return logger
+

results/sacred/10gen_protoss/feudal/_sources/main_654daaa6534bcee62784d639ea63e51d.py

@@ -0,0 +1,124 @@
+import random
+
+import numpy as np
+import os
+import collections
+from os.path import dirname, abspath, join
+from copy import deepcopy
+from sacred import Experiment, SETTINGS
+from sacred.observers import FileStorageObserver
+from sacred.utils import apply_backspaces_and_linefeeds
+import sys
+import torch as th
+from utils.logging import get_logger
+import yaml
+import collections.abc
+
+from run import REGISTRY as run_REGISTRY
+
+SETTINGS['CAPTURE_MODE'] = "no"  # set to "no" if you want to see stdout/stderr in console "fd" or "no"
+logger = get_logger()
+
+ex = Experiment("pymarl")
+ex.logger = logger
+ex.captured_out_filter = apply_backspaces_and_linefeeds
+
+results_path = join(dirname(dirname(abspath(__file__))))
+
+
+@ex.main
+def my_main(_run, _config, _log):
+    # Setting the random seed throughout the modules
+    config = config_copy(_config)
+    random.seed(config["seed"])
+    np.random.seed(config["seed"])
+    th.manual_seed(config["seed"])
+    th.cuda.manual_seed(config["seed"])
+    # th.cuda.manual_seed_all(config["seed"])
+    th.backends.cudnn.deterministic = True  # cudnn
+
+
+    config['env_args']['seed'] = config["seed"]
+
+    # run
+    run_REGISTRY[_config['run']](_run, config, _log)
+
+
+def _get_config(params, arg_name, subfolder):
+    config_name = None
+    for _i, _v in enumerate(params):
+        if _v.split("=")[0] == arg_name:
+            config_name = _v.split("=")[1]
+            del params[_i]
+            break
+
+    if config_name is not None:
+        with open(os.path.join(os.path.dirname(__file__), "config", subfolder, "{}.yaml".format(config_name)),
+                  "r") as f:
+            try:
+                config_dict = yaml.load(f, Loader=yaml.SafeLoader)
+            except yaml.YAMLError as exc:
+                assert False, "{}.yaml error: {}".format(config_name, exc)
+        return config_dict
+
+
+def recursive_dict_update(d, u):
+    for k, v in u.items():
+        if isinstance(v, collections.abc.Mapping):
+            d[k] = recursive_dict_update(d.get(k, {}), v)
+        else:
+            d[k] = v
+    return d
+
+
+def config_copy(config):
+    if isinstance(config, dict):
+        return {k: config_copy(v) for k, v in config.items()}
+    elif isinstance(config, list):
+        return [config_copy(v) for v in config]
+    else:
+        return deepcopy(config)
+
+
+def parse_command(params, key, default):
+    result = default
+    for _i, _v in enumerate(params):
+        if _v.split("=")[0].strip() == key:
+            result = _v[_v.index('=') + 1:].strip()
+            break
+    return result
+
+
+if __name__ == '__main__':
+    params = deepcopy(sys.argv)
+
+    # Get the defaults from default.yaml
+    with open(os.path.join(os.path.dirname(__file__), "config", "default.yaml"), "r") as f:
+        try:
+            config_dict = yaml.load(f, Loader=yaml.SafeLoader)
+        except yaml.YAMLError as exc:
+            assert False, "default.yaml error: {}".format(exc)
+
+    # Load algorithm and env base configs
+    env_config = _get_config(params, "--env-config", "envs")
+    alg_config = _get_config(params, "--config", "algs")
+    # config_dict = {**config_dict, **env_config, **alg_config}
+    config_dict = recursive_dict_update(config_dict, env_config)
+    config_dict = recursive_dict_update(config_dict, alg_config)
+
+    # now add all the config to sacred
+    ex.add_config(config_dict)
+
+    # Save to disk by default for sacred
+    map_name = parse_command(params, "env_args.map_name", config_dict['env_args']['map_name'])
+    algo_name = parse_command(params, "name", config_dict['name'])
+    local_results_path = parse_command(params, "local_results_path", config_dict['local_results_path'])
+    file_obs_path = join(results_path, local_results_path, "sacred", map_name, algo_name)
+
+    logger.info("Saving to FileStorageObserver in {}.".format(file_obs_path))
+    ex.observers.append(FileStorageObserver.create(file_obs_path))
+
+    ex.run_commandline(params)
+
+    # flush
+    sys.stdout.flush()

results/sacred/10gen_protoss/qmix/1/config.json

@@ -0,0 +1,130 @@
+{
+  "action_selector": "epsilon_greedy",
+  "agent": "n_rnn",
+  "agent_output_type": "q",
+  "asn_hidden_size": 32,
+  "batch_size": 128,
+  "batch_size_run": 4,
+  "buffer_cpu_only": true,
+  "buffer_size": 5000,
+  "checkpoint_path": "",
+  "cpu_inference": true,
+  "critic_lr": 0.0005,
+  "double_q": true,
+  "enable_parallel_computing": false,
+  "env": "sc2_v2",
+  "env_args": {
+    "capability_config": {
+      "n_units": 5,
+      "start_positions": {
+        "dist_type": "surrounded_and_reflect",
+        "map_x": 32,
+        "map_y": 32,
+        "n_enemies": 5,
+        "p": 0.5
+      },
+      "team_gen": {
+        "dist_type": "weighted_teams",
+        "exception_unit_types": [
+          "colossus"
+        ],
+        "observe": true,
+        "unit_types": [
+          "stalker",
+          "zealot",
+          "colossus"
+        ],
+        "weights": [
+          0.45,
+          0.45,
+          0.1
+        ]
+      }
+    },
+    "change_fov_with_move": false,
+    "conic_fov": false,
+    "continuing_episode": false,
+    "debug": false,
+    "difficulty": "7",
+    "fully_observable": false,
+    "game_version": null,
+    "heuristic_ai": false,
+    "heuristic_rest": false,
+    "kill_unit_step_mul": 2,
+    "map_name": "10gen_protoss",
+    "move_amount": 2,
+    "num_fov_actions": 12,
+    "obs_all_health": true,
+    "obs_instead_of_state": false,
+    "obs_last_action": false,
+    "obs_own_health": true,
+    "obs_own_pos": true,
+    "obs_pathing_grid": false,
+    "obs_starcraft": true,
+    "obs_terrain_height": false,
+    "obs_timestep_number": false,
+    "replay_dir": "",
+    "replay_prefix": "",
+    "reward_death_value": 10,
+    "reward_defeat": 0,
+    "reward_negative_scale": 0.5,
+    "reward_only_positive": true,
+    "reward_scale": true,
+    "reward_scale_rate": 20,
+    "reward_sparse": false,
+    "reward_win": 200,
+    "seed": null,
+    "state_last_action": true,
+    "state_timestep_number": false,
+    "step_mul": 8
+  },
+  "epsilon_anneal_time": 100000,
+  "epsilon_finish": 0.05,
+  "epsilon_start": 1.0,
+  "evaluate": false,
+  "gain": 0.01,
+  "gamma": 0.99,
+  "grad_norm_clip": 10,
+  "hypernet_embed": 64,
+  "label": "default_label",
+  "learner": "nq_learner",
+  "learner_log_interval": 10000,
+  "load_step": 0,
+  "local_results_path": "results",
+  "log_interval": 10000,
+  "lr": 0.001,
+  "mac": "n_mac",
+  "mixer": "qmix",
+  "mixing_embed_dim": 32,
+  "name": "qmix",
+  "obs_agent_id": true,
+  "obs_last_action": false,
+  "optim_alpha": 0.99,
+  "optim_eps": 1e-05,
+  "optimizer": "adam",
+  "per_alpha": 0.6,
+  "per_beta": 0.4,
+  "q_lambda": false,
+  "repeat_id": 1,
+  "return_priority": false,
+  "rnn_hidden_dim": 64,
+  "run": "default",
+  "runner": "parallel",
+  "runner_log_interval": 10000,
+  "save_model": true,
+  "save_model_interval": 2000000,
+  "save_replay": false,
+  "seed": 163470727,
+  "t_max": 4050000,
+  "target_update_interval": 200,
+  "td_lambda": 0.6,
+  "test_greedy": true,
+  "test_interval": 10000,
+  "test_nepisode": 32,
+  "thread_num": 4,
+  "use_cuda": true,
+  "use_layer_norm": false,
+  "use_orthogonal": false,
+  "use_per": false,
+  "use_tensorboard": true
+}

results/sacred/10gen_protoss/qmix/1/metrics.json

@@ -0,0 +1 @@
+{}

results/sacred/10gen_protoss/qmix/1/run.json

@@ -0,0 +1,124 @@
+{
+  "artifacts": [],
+  "command": "my_main",
+  "experiment": {
+    "base_dir": "C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src",
+    "dependencies": [
+      "numpy==1.23.1",
+      "PyYAML==6.0.2",
+      "sacred==0.8.7",
+      "torch==1.13.1+cu117"
+    ],
+    "mainfile": "main.py",
+    "name": "pymarl",
+    "repositories": [
+      {
+        "commit": "44bb188185fd0292a1a306c86396027efb44224d",
+        "dirty": true,
+        "url": "https://github.com/tjuHaoXiaotian/pymarl3.git"
+      },
+      {
+        "commit": "44bb188185fd0292a1a306c86396027efb44224d",
+        "dirty": true,
+        "url": "https://github.com/tjuHaoXiaotian/pymarl3.git"
+      }
+    ],
+    "sources": [
+      [
+        "main.py",
+        "_sources\\main_654daaa6534bcee62784d639ea63e51d.py"
+      ],
+      [
+        "utils\\logging.py",
+        "_sources\\logging_f71df6d788e929fac28afdf951d63d54.py"
+      ]
+    ]
+  },
+  "fail_trace": [
+    "Traceback (most recent call last):\n",
+    "  File \"C:\\Users\\Taiyo\\.conda\\envs\\SMACV2\\lib\\site-packages\\sacred\\config\\captured_function.py\", line 42, in captured_function\n    result = wrapped(*args, **kwargs)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\main.py\", line 44, in my_main\n    run_REGISTRY[_config['run']](_run, config, _log)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\run\\run.py\", line 122, in run\n    logger.setup_tb(tb_exp_direc)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\utils\\logging.py\", line 19, in setup_tb\n    configure(directory_name)\n",
+    "  File \"C:\\Users\\Taiyo\\.conda\\envs\\SMACV2\\lib\\site-packages\\tensorboard_logger\\tensorboard_logger.py\", line 296, in configure\n    _default_logger = Logger(logdir, flush_secs=flush_secs)\n",
+    "  File \"C:\\Users\\Taiyo\\.conda\\envs\\SMACV2\\lib\\site-packages\\tensorboard_logger\\tensorboard_logger.py\", line 51, in __init__\n    self._writer = open(filename, 'wb')\n",
+    "FileNotFoundError: [Errno 2] No such file or directory: 'C:\\\\Users\\\\Taiyo\\\\Desktop\\\\SMAC V2\\\\pymarl3\\\\results\\\\tb_logs\\\\sc2_v2_10gen_protoss-obs_aid=1-obs_act=0-conic_fov=0\\\\algo=qmix-agent=n_rnn\\\\env_n=4\\\\mixer=qmix\\\\rnn_dim=64-2bs=5000_128-tdlambda=0.6-epdec_0.05=100k\\\\qmix__2025-01-06_00-15-29\\\\events.out.tfevents.1736093730.Taiyopen'\n"
+  ],
+  "heartbeat": "2025-01-05T16:15:30.002599",
+  "host": {
+    "ENV": {},
+    "cpu": "AMD Ryzen 7 5700X3D 8-Core Processor",
+    "gpus": {
+      "driver_version": "560.94",
+      "gpus": [
+        {
+          "model": "NVIDIA GeForce RTX 4080 SUPER",
+          "persistence_mode": false,
+          "total_memory": 16376
+        }
+      ]
+    },
+    "hostname": "Taiyopen",
+    "os": [
+      "Windows",
+      "Windows-10-10.0.22631-SP0"
+    ],
+    "python_version": "3.10.16"
+  },
+  "meta": {
+    "command": "my_main",
+    "config_updates": {
+      "batch_size": 128,
+      "batch_size_run": 4,
+      "buffer_size": 5000,
+      "epsilon_anneal_time": 100000,
+      "obs_agent_id": true,
+      "obs_last_action": false,
+      "runner": "parallel",
+      "t_max": 4050000,
+      "td_lambda": 0.6
+    },
+    "named_configs": [],
+    "options": {
+      "--beat-interval": null,
+      "--capture": null,
+      "--comment": null,
+      "--debug": false,
+      "--enforce_clean": false,
+      "--file_storage": null,
+      "--force": false,
+      "--help": false,
+      "--id": null,
+      "--loglevel": null,
+      "--mongo_db": null,
+      "--name": null,
+      "--pdb": false,
+      "--print-config": false,
+      "--priority": null,
+      "--queue": false,
+      "--s3": null,
+      "--sql": null,
+      "--tiny_db": null,
+      "--unobserved": false,
+      "COMMAND": null,
+      "UPDATE": [
+        "obs_agent_id=True",
+        "obs_last_action=False",
+        "runner=parallel",
+        "batch_size_run=4",
+        "buffer_size=5000",
+        "t_max=4050000",
+        "epsilon_anneal_time=100000",
+        "batch_size=128",
+        "td_lambda=0.6"
+      ],
+      "help": false,
+      "with": true
+    }
+  },
+  "resources": [],
+  "result": null,
+  "start_time": "2025-01-05T16:15:29.896110",
+  "status": "FAILED",
+  "stop_time": "2025-01-05T16:15:30.004651"
+}

results/sacred/10gen_protoss/qmix/2/config.json

@@ -0,0 +1,130 @@
+{
+  "action_selector": "epsilon_greedy",
+  "agent": "n_rnn",
+  "agent_output_type": "q",
+  "asn_hidden_size": 32,
+  "batch_size": 128,
+  "batch_size_run": 4,
+  "buffer_cpu_only": true,
+  "buffer_size": 5000,
+  "checkpoint_path": "",
+  "cpu_inference": true,
+  "critic_lr": 0.0005,
+  "double_q": true,
+  "enable_parallel_computing": false,
+  "env": "sc2_v2",
+  "env_args": {
+    "capability_config": {
+      "n_units": 5,
+      "start_positions": {
+        "dist_type": "surrounded_and_reflect",
+        "map_x": 32,
+        "map_y": 32,
+        "n_enemies": 5,
+        "p": 0.5
+      },
+      "team_gen": {
+        "dist_type": "weighted_teams",
+        "exception_unit_types": [
+          "colossus"
+        ],
+        "observe": true,
+        "unit_types": [
+          "stalker",
+          "zealot",
+          "colossus"
+        ],
+        "weights": [
+          0.45,
+          0.45,
+          0.1
+        ]
+      }
+    },
+    "change_fov_with_move": false,
+    "conic_fov": false,
+    "continuing_episode": false,
+    "debug": false,
+    "difficulty": "7",
+    "fully_observable": false,
+    "game_version": null,
+    "heuristic_ai": false,
+    "heuristic_rest": false,
+    "kill_unit_step_mul": 2,
+    "map_name": "10gen_protoss",
+    "move_amount": 2,
+    "num_fov_actions": 12,
+    "obs_all_health": true,
+    "obs_instead_of_state": false,
+    "obs_last_action": false,
+    "obs_own_health": true,
+    "obs_own_pos": true,
+    "obs_pathing_grid": false,
+    "obs_starcraft": true,
+    "obs_terrain_height": false,
+    "obs_timestep_number": false,
+    "replay_dir": "",
+    "replay_prefix": "",
+    "reward_death_value": 10,
+    "reward_defeat": 0,
+    "reward_negative_scale": 0.5,
+    "reward_only_positive": true,
+    "reward_scale": true,
+    "reward_scale_rate": 20,
+    "reward_sparse": false,
+    "reward_win": 200,
+    "seed": null,
+    "state_last_action": true,
+    "state_timestep_number": false,
+    "step_mul": 8
+  },
+  "epsilon_anneal_time": 100000,
+  "epsilon_finish": 0.05,
+  "epsilon_start": 1.0,
+  "evaluate": false,
+  "gain": 0.01,
+  "gamma": 0.99,
+  "grad_norm_clip": 10,
+  "hypernet_embed": 64,
+  "label": "default_label",
+  "learner": "nq_learner",
+  "learner_log_interval": 10000,
+  "load_step": 0,
+  "local_results_path": "results",
+  "log_interval": 10000,
+  "lr": 0.001,
+  "mac": "n_mac",
+  "mixer": "qmix",
+  "mixing_embed_dim": 32,
+  "name": "qmix",
+  "obs_agent_id": true,
+  "obs_last_action": false,
+  "optim_alpha": 0.99,
+  "optim_eps": 1e-05,
+  "optimizer": "adam",
+  "per_alpha": 0.6,
+  "per_beta": 0.4,
+  "q_lambda": false,
+  "repeat_id": 1,
+  "return_priority": false,
+  "rnn_hidden_dim": 64,
+  "run": "default",
+  "runner": "parallel",
+  "runner_log_interval": 10000,
+  "save_model": true,
+  "save_model_interval": 2000000,
+  "save_replay": false,
+  "seed": 790531776,
+  "t_max": 4050000,
+  "target_update_interval": 200,
+  "td_lambda": 0.6,
+  "test_greedy": true,
+  "test_interval": 10000,
+  "test_nepisode": 32,
+  "thread_num": 4,
+  "use_cuda": true,
+  "use_layer_norm": false,
+  "use_orthogonal": false,
+  "use_per": false,
+  "use_tensorboard": true
+}

results/sacred/10gen_protoss/qmix/2/metrics.json

@@ -0,0 +1 @@
+{}

results/sacred/10gen_protoss/qmix/2/run.json

@@ -0,0 +1,124 @@
+{
+  "artifacts": [],
+  "command": "my_main",
+  "experiment": {
+    "base_dir": "C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src",
+    "dependencies": [
+      "numpy==1.23.1",
+      "PyYAML==6.0.2",
+      "sacred==0.8.7",
+      "torch==1.13.1+cu117"
+    ],
+    "mainfile": "main.py",
+    "name": "pymarl",
+    "repositories": [
+      {
+        "commit": "44bb188185fd0292a1a306c86396027efb44224d",
+        "dirty": true,
+        "url": "https://github.com/tjuHaoXiaotian/pymarl3.git"
+      },
+      {
+        "commit": "44bb188185fd0292a1a306c86396027efb44224d",
+        "dirty": true,
+        "url": "https://github.com/tjuHaoXiaotian/pymarl3.git"
+      }
+    ],
+    "sources": [
+      [
+        "main.py",
+        "_sources\\main_654daaa6534bcee62784d639ea63e51d.py"
+      ],
+      [
+        "utils\\logging.py",
+        "_sources\\logging_f71df6d788e929fac28afdf951d63d54.py"
+      ]
+    ]
+  },
+  "fail_trace": [
+    "Traceback (most recent call last):\n",
+    "  File \"C:\\Users\\Taiyo\\.conda\\envs\\SMACV2\\lib\\site-packages\\sacred\\config\\captured_function.py\", line 42, in captured_function\n    result = wrapped(*args, **kwargs)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\main.py\", line 44, in my_main\n    run_REGISTRY[_config['run']](_run, config, _log)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\run\\run.py\", line 122, in run\n    logger.setup_tb(tb_exp_direc)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\utils\\logging.py\", line 19, in setup_tb\n    configure(directory_name)\n",
+    "  File \"C:\\Users\\Taiyo\\.conda\\envs\\SMACV2\\lib\\site-packages\\tensorboard_logger\\tensorboard_logger.py\", line 296, in configure\n    _default_logger = Logger(logdir, flush_secs=flush_secs)\n",
+    "  File \"C:\\Users\\Taiyo\\.conda\\envs\\SMACV2\\lib\\site-packages\\tensorboard_logger\\tensorboard_logger.py\", line 51, in __init__\n    self._writer = open(filename, 'wb')\n",
+    "FileNotFoundError: [Errno 2] No such file or directory: 'C:\\\\Users\\\\Taiyo\\\\Desktop\\\\SMAC V2\\\\pymarl3\\\\results\\\\tb_logs\\\\sc2_v2_10gen_protoss-obs_aid=1-obs_act=0-conic_fov=0\\\\algo=qmix-agent=n_rnn\\\\env_n=4\\\\mixer=qmix\\\\rnn_dim=64-2bs=5000_128-tdlambda=0.6-epdec_0.05=100k\\\\qmix__2025-01-06_00-15-40\\\\events.out.tfevents.1736093740.Taiyopen'\n"
+  ],
+  "heartbeat": "2025-01-05T16:15:40.344931",
+  "host": {
+    "ENV": {},
+    "cpu": "AMD Ryzen 7 5700X3D 8-Core Processor",
+    "gpus": {
+      "driver_version": "560.94",
+      "gpus": [
+        {
+          "model": "NVIDIA GeForce RTX 4080 SUPER",
+          "persistence_mode": false,
+          "total_memory": 16376
+        }
+      ]
+    },
+    "hostname": "Taiyopen",
+    "os": [
+      "Windows",
+      "Windows-10-10.0.22631-SP0"
+    ],
+    "python_version": "3.10.16"
+  },
+  "meta": {
+    "command": "my_main",
+    "config_updates": {
+      "batch_size": 128,
+      "batch_size_run": 4,
+      "buffer_size": 5000,
+      "epsilon_anneal_time": 100000,
+      "obs_agent_id": true,
+      "obs_last_action": false,
+      "runner": "parallel",
+      "t_max": 4050000,
+      "td_lambda": 0.6
+    },
+    "named_configs": [],
+    "options": {
+      "--beat-interval": null,
+      "--capture": null,
+      "--comment": null,
+      "--debug": false,
+      "--enforce_clean": false,
+      "--file_storage": null,
+      "--force": false,
+      "--help": false,
+      "--id": null,
+      "--loglevel": null,
+      "--mongo_db": null,
+      "--name": null,
+      "--pdb": false,
+      "--print-config": false,
+      "--priority": null,
+      "--queue": false,
+      "--s3": null,
+      "--sql": null,
+      "--tiny_db": null,
+      "--unobserved": false,
+      "COMMAND": null,
+      "UPDATE": [
+        "obs_agent_id=True",
+        "obs_last_action=False",
+        "runner=parallel",
+        "batch_size_run=4",
+        "buffer_size=5000",
+        "t_max=4050000",
+        "epsilon_anneal_time=100000",
+        "batch_size=128",
+        "td_lambda=0.6"
+      ],
+      "help": false,
+      "with": true
+    }
+  },
+  "resources": [],
+  "result": null,
+  "start_time": "2025-01-05T16:15:40.272388",
+  "status": "FAILED",
+  "stop_time": "2025-01-05T16:15:40.346943"
+}

results/sacred/10gen_protoss/qmix/3/config.json

@@ -0,0 +1,130 @@
+{
+  "action_selector": "epsilon_greedy",
+  "agent": "n_rnn",
+  "agent_output_type": "q",
+  "asn_hidden_size": 32,
+  "batch_size": 128,
+  "batch_size_run": 4,
+  "buffer_cpu_only": true,
+  "buffer_size": 5000,
+  "checkpoint_path": "",
+  "cpu_inference": true,
+  "critic_lr": 0.0005,
+  "double_q": true,
+  "enable_parallel_computing": false,
+  "env": "sc2_v2",
+  "env_args": {
+    "capability_config": {
+      "n_units": 5,
+      "start_positions": {
+        "dist_type": "surrounded_and_reflect",
+        "map_x": 32,
+        "map_y": 32,
+        "n_enemies": 5,
+        "p": 0.5
+      },
+      "team_gen": {
+        "dist_type": "weighted_teams",
+        "exception_unit_types": [
+          "colossus"
+        ],
+        "observe": true,
+        "unit_types": [
+          "stalker",
+          "zealot",
+          "colossus"
+        ],
+        "weights": [
+          0.45,
+          0.45,
+          0.1
+        ]
+      }
+    },
+    "change_fov_with_move": false,
+    "conic_fov": false,
+    "continuing_episode": false,
+    "debug": false,
+    "difficulty": "7",
+    "fully_observable": false,
+    "game_version": null,
+    "heuristic_ai": false,
+    "heuristic_rest": false,
+    "kill_unit_step_mul": 2,
+    "map_name": "10gen_protoss",
+    "move_amount": 2,
+    "num_fov_actions": 12,
+    "obs_all_health": true,
+    "obs_instead_of_state": false,
+    "obs_last_action": false,
+    "obs_own_health": true,
+    "obs_own_pos": true,
+    "obs_pathing_grid": false,
+    "obs_starcraft": true,
+    "obs_terrain_height": false,
+    "obs_timestep_number": false,
+    "replay_dir": "",
+    "replay_prefix": "",
+    "reward_death_value": 10,
+    "reward_defeat": 0,
+    "reward_negative_scale": 0.5,
+    "reward_only_positive": true,
+    "reward_scale": true,
+    "reward_scale_rate": 20,
+    "reward_sparse": false,
+    "reward_win": 200,
+    "seed": null,
+    "state_last_action": true,
+    "state_timestep_number": false,
+    "step_mul": 8
+  },
+  "epsilon_anneal_time": 100000,
+  "epsilon_finish": 0.05,
+  "epsilon_start": 1.0,
+  "evaluate": false,
+  "gain": 0.01,
+  "gamma": 0.99,
+  "grad_norm_clip": 10,
+  "hypernet_embed": 64,
+  "label": "default_label",
+  "learner": "nq_learner",
+  "learner_log_interval": 10000,
+  "load_step": 0,
+  "local_results_path": "results",
+  "log_interval": 10000,
+  "lr": 0.001,
+  "mac": "n_mac",
+  "mixer": "qmix",
+  "mixing_embed_dim": 32,
+  "name": "qmix",
+  "obs_agent_id": true,
+  "obs_last_action": false,
+  "optim_alpha": 0.99,
+  "optim_eps": 1e-05,
+  "optimizer": "adam",
+  "per_alpha": 0.6,
+  "per_beta": 0.4,
+  "q_lambda": false,
+  "repeat_id": 1,
+  "return_priority": false,
+  "rnn_hidden_dim": 64,
+  "run": "default",
+  "runner": "parallel",
+  "runner_log_interval": 10000,
+  "save_model": true,
+  "save_model_interval": 2000000,
+  "save_replay": false,
+  "seed": 788463946,
+  "t_max": 4050000,
+  "target_update_interval": 200,
+  "td_lambda": 0.6,
+  "test_greedy": true,
+  "test_interval": 10000,
+  "test_nepisode": 32,
+  "thread_num": 4,
+  "use_cuda": true,
+  "use_layer_norm": false,
+  "use_orthogonal": false,
+  "use_per": false,
+  "use_tensorboard": true
+}

results/sacred/10gen_protoss/qmix/3/metrics.json

@@ -0,0 +1 @@
+{}

results/sacred/10gen_protoss/qmix/3/run.json

@@ -0,0 +1,124 @@
+{
+  "artifacts": [],
+  "command": "my_main",
+  "experiment": {
+    "base_dir": "C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src",
+    "dependencies": [
+      "numpy==1.23.1",
+      "PyYAML==6.0.2",
+      "sacred==0.8.7",
+      "torch==1.13.1+cu117"
+    ],
+    "mainfile": "main.py",
+    "name": "pymarl",
+    "repositories": [
+      {
+        "commit": "44bb188185fd0292a1a306c86396027efb44224d",
+        "dirty": true,
+        "url": "https://github.com/tjuHaoXiaotian/pymarl3.git"
+      },
+      {
+        "commit": "44bb188185fd0292a1a306c86396027efb44224d",
+        "dirty": true,
+        "url": "https://github.com/tjuHaoXiaotian/pymarl3.git"
+      }
+    ],
+    "sources": [
+      [
+        "main.py",
+        "_sources\\main_654daaa6534bcee62784d639ea63e51d.py"
+      ],
+      [
+        "utils\\logging.py",
+        "_sources\\logging_f71df6d788e929fac28afdf951d63d54.py"
+      ]
+    ]
+  },
+  "fail_trace": [
+    "Traceback (most recent call last):\n",
+    "  File \"C:\\Users\\Taiyo\\.conda\\envs\\SMACV2\\lib\\site-packages\\sacred\\config\\captured_function.py\", line 42, in captured_function\n    result = wrapped(*args, **kwargs)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\main.py\", line 44, in my_main\n    run_REGISTRY[_config['run']](_run, config, _log)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\run\\run.py\", line 122, in run\n    logger.setup_tb(tb_exp_direc)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\utils\\logging.py\", line 19, in setup_tb\n    configure(directory_name)\n",
+    "  File \"C:\\Users\\Taiyo\\.conda\\envs\\SMACV2\\lib\\site-packages\\tensorboard_logger\\tensorboard_logger.py\", line 296, in configure\n    _default_logger = Logger(logdir, flush_secs=flush_secs)\n",
+    "  File \"C:\\Users\\Taiyo\\.conda\\envs\\SMACV2\\lib\\site-packages\\tensorboard_logger\\tensorboard_logger.py\", line 51, in __init__\n    self._writer = open(filename, 'wb')\n",
+    "FileNotFoundError: [Errno 2] No such file or directory: 'C:\\\\Users\\\\Taiyo\\\\Desktop\\\\SMAC V2\\\\pymarl3\\\\results\\\\tb_logs\\\\sc2_v2_10gen_protoss-obs_aid=1-obs_act=0-conic_fov=0\\\\algo=qmix-agent=n_rnn\\\\env_n=4\\\\mixer=qmix\\\\rnn_dim=64-2bs=5000_128-tdlambda=0.6-epdec_0.05=100k\\\\qmix__2025-01-06_00-16-48\\\\events.out.tfevents.1736093809.Taiyopen'\n"
+  ],
+  "heartbeat": "2025-01-05T16:16:49.016218",
+  "host": {
+    "ENV": {},
+    "cpu": "AMD Ryzen 7 5700X3D 8-Core Processor",
+    "gpus": {
+      "driver_version": "560.94",
+      "gpus": [
+        {
+          "model": "NVIDIA GeForce RTX 4080 SUPER",
+          "persistence_mode": false,
+          "total_memory": 16376
+        }
+      ]
+    },
+    "hostname": "Taiyopen",
+    "os": [
+      "Windows",
+      "Windows-10-10.0.22631-SP0"
+    ],
+    "python_version": "3.10.16"
+  },
+  "meta": {
+    "command": "my_main",
+    "config_updates": {
+      "batch_size": 128,
+      "batch_size_run": 4,
+      "buffer_size": 5000,
+      "epsilon_anneal_time": 100000,
+      "obs_agent_id": true,
+      "obs_last_action": false,
+      "runner": "parallel",
+      "t_max": 4050000,
+      "td_lambda": 0.6
+    },
+    "named_configs": [],
+    "options": {
+      "--beat-interval": null,
+      "--capture": null,
+      "--comment": null,
+      "--debug": false,
+      "--enforce_clean": false,
+      "--file_storage": null,
+      "--force": false,
+      "--help": false,
+      "--id": null,
+      "--loglevel": null,
+      "--mongo_db": null,
+      "--name": null,
+      "--pdb": false,
+      "--print-config": false,
+      "--priority": null,
+      "--queue": false,
+      "--s3": null,
+      "--sql": null,
+      "--tiny_db": null,
+      "--unobserved": false,
+      "COMMAND": null,
+      "UPDATE": [
+        "obs_agent_id=True",
+        "obs_last_action=False",
+        "runner=parallel",
+        "batch_size_run=4",
+        "buffer_size=5000",
+        "t_max=4050000",
+        "epsilon_anneal_time=100000",
+        "batch_size=128",
+        "td_lambda=0.6"
+      ],
+      "help": false,
+      "with": true
+    }
+  },
+  "resources": [],
+  "result": null,
+  "start_time": "2025-01-05T16:16:48.931426",
+  "status": "FAILED",
+  "stop_time": "2025-01-05T16:16:49.018225"
+}

results/sacred/10gen_protoss/qmix/4/config.json

@@ -0,0 +1,130 @@
+{
+  "action_selector": "epsilon_greedy",
+  "agent": "n_rnn",
+  "agent_output_type": "q",
+  "asn_hidden_size": 32,
+  "batch_size": 128,
+  "batch_size_run": 4,
+  "buffer_cpu_only": true,
+  "buffer_size": 5000,
+  "checkpoint_path": "",
+  "cpu_inference": true,
+  "critic_lr": 0.0005,
+  "double_q": true,
+  "enable_parallel_computing": false,
+  "env": "sc2_v2",
+  "env_args": {
+    "capability_config": {
+      "n_units": 5,
+      "start_positions": {
+        "dist_type": "surrounded_and_reflect",
+        "map_x": 32,
+        "map_y": 32,
+        "n_enemies": 5,
+        "p": 0.5
+      },
+      "team_gen": {
+        "dist_type": "weighted_teams",
+        "exception_unit_types": [
+          "colossus"
+        ],
+        "observe": true,
+        "unit_types": [
+          "stalker",
+          "zealot",
+          "colossus"
+        ],
+        "weights": [
+          0.45,
+          0.45,
+          0.1
+        ]
+      }
+    },
+    "change_fov_with_move": false,
+    "conic_fov": false,
+    "continuing_episode": false,
+    "debug": false,
+    "difficulty": "7",
+    "fully_observable": false,
+    "game_version": null,
+    "heuristic_ai": false,
+    "heuristic_rest": false,
+    "kill_unit_step_mul": 2,
+    "map_name": "10gen_protoss",
+    "move_amount": 2,
+    "num_fov_actions": 12,
+    "obs_all_health": true,
+    "obs_instead_of_state": false,
+    "obs_last_action": false,
+    "obs_own_health": true,
+    "obs_own_pos": true,
+    "obs_pathing_grid": false,
+    "obs_starcraft": true,
+    "obs_terrain_height": false,
+    "obs_timestep_number": false,
+    "replay_dir": "",
+    "replay_prefix": "",
+    "reward_death_value": 10,
+    "reward_defeat": 0,
+    "reward_negative_scale": 0.5,
+    "reward_only_positive": true,
+    "reward_scale": true,
+    "reward_scale_rate": 20,
+    "reward_sparse": false,
+    "reward_win": 200,
+    "seed": null,
+    "state_last_action": true,
+    "state_timestep_number": false,
+    "step_mul": 8
+  },
+  "epsilon_anneal_time": 100000,
+  "epsilon_finish": 0.05,
+  "epsilon_start": 1.0,
+  "evaluate": false,
+  "gain": 0.01,
+  "gamma": 0.99,
+  "grad_norm_clip": 10,
+  "hypernet_embed": 64,
+  "label": "default_label",
+  "learner": "nq_learner",
+  "learner_log_interval": 10000,
+  "load_step": 0,
+  "local_results_path": "results",
+  "log_interval": 10000,
+  "lr": 0.001,
+  "mac": "n_mac",
+  "mixer": "qmix",
+  "mixing_embed_dim": 32,
+  "name": "qmix",
+  "obs_agent_id": true,
+  "obs_last_action": false,
+  "optim_alpha": 0.99,
+  "optim_eps": 1e-05,
+  "optimizer": "adam",
+  "per_alpha": 0.6,
+  "per_beta": 0.4,
+  "q_lambda": false,
+  "repeat_id": 1,
+  "return_priority": false,
+  "rnn_hidden_dim": 64,
+  "run": "default",
+  "runner": "parallel",
+  "runner_log_interval": 10000,
+  "save_model": true,
+  "save_model_interval": 2000000,
+  "save_replay": false,
+  "seed": 727581499,
+  "t_max": 4050000,
+  "target_update_interval": 200,
+  "td_lambda": 0.6,
+  "test_greedy": true,
+  "test_interval": 10000,
+  "test_nepisode": 32,
+  "thread_num": 4,
+  "use_cuda": true,
+  "use_layer_norm": false,
+  "use_orthogonal": false,
+  "use_per": false,
+  "use_tensorboard": true
+}

results/sacred/10gen_protoss/qmix/4/metrics.json

@@ -0,0 +1 @@
+{}

results/sacred/10gen_protoss/qmix/4/run.json

@@ -0,0 +1,124 @@
+{
+  "artifacts": [],
+  "command": "my_main",
+  "experiment": {
+    "base_dir": "C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src",
+    "dependencies": [
+      "numpy==1.23.1",
+      "PyYAML==6.0.2",
+      "sacred==0.8.7",
+      "torch==1.13.1+cu117"
+    ],
+    "mainfile": "main.py",
+    "name": "pymarl",
+    "repositories": [
+      {
+        "commit": "44bb188185fd0292a1a306c86396027efb44224d",
+        "dirty": true,
+        "url": "https://github.com/tjuHaoXiaotian/pymarl3.git"
+      },
+      {
+        "commit": "44bb188185fd0292a1a306c86396027efb44224d",
+        "dirty": true,
+        "url": "https://github.com/tjuHaoXiaotian/pymarl3.git"
+      }
+    ],
+    "sources": [
+      [
+        "main.py",
+        "_sources\\main_654daaa6534bcee62784d639ea63e51d.py"
+      ],
+      [
+        "utils\\logging.py",
+        "_sources\\logging_f71df6d788e929fac28afdf951d63d54.py"
+      ]
+    ]
+  },
+  "fail_trace": [
+    "Traceback (most recent call last):\n",
+    "  File \"C:\\Users\\Taiyo\\.conda\\envs\\SMACV2\\lib\\site-packages\\sacred\\config\\captured_function.py\", line 42, in captured_function\n    result = wrapped(*args, **kwargs)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\main.py\", line 44, in my_main\n    run_REGISTRY[_config['run']](_run, config, _log)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\run\\run.py\", line 122, in run\n    logger.setup_tb(tb_exp_direc)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\utils\\logging.py\", line 19, in setup_tb\n    configure(directory_name)\n",
+    "  File \"C:\\Users\\Taiyo\\.conda\\envs\\SMACV2\\lib\\site-packages\\tensorboard_logger\\tensorboard_logger.py\", line 296, in configure\n    _default_logger = Logger(logdir, flush_secs=flush_secs)\n",
+    "  File \"C:\\Users\\Taiyo\\.conda\\envs\\SMACV2\\lib\\site-packages\\tensorboard_logger\\tensorboard_logger.py\", line 51, in __init__\n    self._writer = open(filename, 'wb')\n",
+    "FileNotFoundError: [Errno 2] No such file or directory: 'C:\\\\Users\\\\Taiyo\\\\Desktop\\\\SMAC V2\\\\pymarl3\\\\results\\\\tb_logs\\\\sc2_v2_10gen_protoss-obs_aid=1-obs_act=0-conic_fov=0\\\\algo=qmix-agent=n_rnn\\\\env_n=4\\\\mixer=qmix\\\\rnn_dim=64-2bs=5000_128-tdlambda=0.6-epdec_0.05=100k\\\\qmix__2025-01-06_00-23-20\\\\events.out.tfevents.1736094200.Taiyopen'\n"
+  ],
+  "heartbeat": "2025-01-05T16:23:20.246737",
+  "host": {
+    "ENV": {},
+    "cpu": "AMD Ryzen 7 5700X3D 8-Core Processor",
+    "gpus": {
+      "driver_version": "560.94",
+      "gpus": [
+        {
+          "model": "NVIDIA GeForce RTX 4080 SUPER",
+          "persistence_mode": false,
+          "total_memory": 16376
+        }
+      ]
+    },
+    "hostname": "Taiyopen",
+    "os": [
+      "Windows",
+      "Windows-10-10.0.22631-SP0"
+    ],
+    "python_version": "3.10.16"
+  },
+  "meta": {
+    "command": "my_main",
+    "config_updates": {
+      "batch_size": 128,
+      "batch_size_run": 4,
+      "buffer_size": 5000,
+      "epsilon_anneal_time": 100000,
+      "obs_agent_id": true,
+      "obs_last_action": false,
+      "runner": "parallel",
+      "t_max": 4050000,
+      "td_lambda": 0.6
+    },
+    "named_configs": [],
+    "options": {
+      "--beat-interval": null,
+      "--capture": null,
+      "--comment": null,
+      "--debug": false,
+      "--enforce_clean": false,
+      "--file_storage": null,
+      "--force": false,
+      "--help": false,
+      "--id": null,
+      "--loglevel": null,
+      "--mongo_db": null,
+      "--name": null,
+      "--pdb": false,
+      "--print-config": false,
+      "--priority": null,
+      "--queue": false,
+      "--s3": null,
+      "--sql": null,
+      "--tiny_db": null,
+      "--unobserved": false,
+      "COMMAND": null,
+      "UPDATE": [
+        "obs_agent_id=True",
+        "obs_last_action=False",
+        "runner=parallel",
+        "batch_size_run=4",
+        "buffer_size=5000",
+        "t_max=4050000",
+        "epsilon_anneal_time=100000",
+        "batch_size=128",
+        "td_lambda=0.6"
+      ],
+      "help": false,
+      "with": true
+    }
+  },
+  "resources": [],
+  "result": null,
+  "start_time": "2025-01-05T16:23:20.174349",
+  "status": "FAILED",
+  "stop_time": "2025-01-05T16:23:20.249243"
+}

results/sacred/10gen_protoss/qmix/5/config.json

@@ -0,0 +1,130 @@
+{
+  "action_selector": "epsilon_greedy",
+  "agent": "n_rnn",
+  "agent_output_type": "q",
+  "asn_hidden_size": 32,
+  "batch_size": 128,
+  "batch_size_run": 4,
+  "buffer_cpu_only": true,
+  "buffer_size": 5000,
+  "checkpoint_path": "",
+  "cpu_inference": true,
+  "critic_lr": 0.0005,
+  "double_q": true,
+  "enable_parallel_computing": false,
+  "env": "sc2_v2",
+  "env_args": {
+    "capability_config": {
+      "n_units": 5,
+      "start_positions": {
+        "dist_type": "surrounded_and_reflect",
+        "map_x": 32,
+        "map_y": 32,
+        "n_enemies": 5,
+        "p": 0.5
+      },
+      "team_gen": {
+        "dist_type": "weighted_teams",
+        "exception_unit_types": [
+          "colossus"
+        ],
+        "observe": true,
+        "unit_types": [
+          "stalker",
+          "zealot",
+          "colossus"
+        ],
+        "weights": [
+          0.45,
+          0.45,
+          0.1
+        ]
+      }
+    },
+    "change_fov_with_move": false,
+    "conic_fov": false,
+    "continuing_episode": false,
+    "debug": false,
+    "difficulty": "7",
+    "fully_observable": false,
+    "game_version": null,
+    "heuristic_ai": false,
+    "heuristic_rest": false,
+    "kill_unit_step_mul": 2,
+    "map_name": "10gen_protoss",
+    "move_amount": 2,
+    "num_fov_actions": 12,
+    "obs_all_health": true,
+    "obs_instead_of_state": false,
+    "obs_last_action": false,
+    "obs_own_health": true,
+    "obs_own_pos": true,
+    "obs_pathing_grid": false,
+    "obs_starcraft": true,
+    "obs_terrain_height": false,
+    "obs_timestep_number": false,
+    "replay_dir": "",
+    "replay_prefix": "",
+    "reward_death_value": 10,
+    "reward_defeat": 0,
+    "reward_negative_scale": 0.5,
+    "reward_only_positive": true,
+    "reward_scale": true,
+    "reward_scale_rate": 20,
+    "reward_sparse": false,
+    "reward_win": 200,
+    "seed": null,
+    "state_last_action": true,
+    "state_timestep_number": false,
+    "step_mul": 8
+  },
+  "epsilon_anneal_time": 100000,
+  "epsilon_finish": 0.05,
+  "epsilon_start": 1.0,
+  "evaluate": false,
+  "gain": 0.01,
+  "gamma": 0.99,
+  "grad_norm_clip": 10,
+  "hypernet_embed": 64,
+  "label": "default_label",
+  "learner": "nq_learner",
+  "learner_log_interval": 10000,
+  "load_step": 0,
+  "local_results_path": "results",
+  "log_interval": 10000,
+  "lr": 0.001,
+  "mac": "n_mac",
+  "mixer": "qmix",
+  "mixing_embed_dim": 32,
+  "name": "qmix",
+  "obs_agent_id": true,
+  "obs_last_action": false,
+  "optim_alpha": 0.99,
+  "optim_eps": 1e-05,
+  "optimizer": "adam",
+  "per_alpha": 0.6,
+  "per_beta": 0.4,
+  "q_lambda": false,
+  "repeat_id": 1,
+  "return_priority": false,
+  "rnn_hidden_dim": 64,
+  "run": "default",
+  "runner": "parallel",
+  "runner_log_interval": 10000,
+  "save_model": true,
+  "save_model_interval": 2000000,
+  "save_replay": false,
+  "seed": 148457271,
+  "t_max": 4050000,
+  "target_update_interval": 200,
+  "td_lambda": 0.6,
+  "test_greedy": true,
+  "test_interval": 10000,
+  "test_nepisode": 32,
+  "thread_num": 4,
+  "use_cuda": true,
+  "use_layer_norm": false,
+  "use_orthogonal": false,
+  "use_per": false,
+  "use_tensorboard": true
+}

results/sacred/10gen_protoss/qmix/5/metrics.json

@@ -0,0 +1 @@
+{}

results/sacred/10gen_protoss/qmix/5/run.json

@@ -0,0 +1,126 @@
+{
+  "artifacts": [],
+  "command": "my_main",
+  "experiment": {
+    "base_dir": "C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src",
+    "dependencies": [
+      "numpy==1.23.1",
+      "PyYAML==6.0.2",
+      "sacred==0.8.7",
+      "torch==1.13.1+cu117"
+    ],
+    "mainfile": "main.py",
+    "name": "pymarl",
+    "repositories": [
+      {
+        "commit": "44bb188185fd0292a1a306c86396027efb44224d",
+        "dirty": true,
+        "url": "https://github.com/tjuHaoXiaotian/pymarl3.git"
+      },
+      {
+        "commit": "44bb188185fd0292a1a306c86396027efb44224d",
+        "dirty": true,
+        "url": "https://github.com/tjuHaoXiaotian/pymarl3.git"
+      }
+    ],
+    "sources": [
+      [
+        "main.py",
+        "_sources\\main_654daaa6534bcee62784d639ea63e51d.py"
+      ],
+      [
+        "utils\\logging.py",
+        "_sources\\logging_f71df6d788e929fac28afdf951d63d54.py"
+      ]
+    ]
+  },
+  "fail_trace": [
+    "Traceback (most recent call last):\n",
+    "  File \"C:\\Users\\Taiyo\\.conda\\envs\\SMACV2\\lib\\site-packages\\sacred\\config\\captured_function.py\", line 42, in captured_function\n    result = wrapped(*args, **kwargs)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\main.py\", line 44, in my_main\n    run_REGISTRY[_config['run']](_run, config, _log)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\run\\run.py\", line 122, in run\n    logger.setup_tb(tb_exp_direc)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\utils\\logging.py\", line 19, in setup_tb\n    configure(directory_name)\n",
+    "  File \"C:\\Users\\Taiyo\\.conda\\envs\\SMACV2\\lib\\site-packages\\tensorboard_logger\\tensorboard_logger.py\", line 296, in configure\n    _default_logger = Logger(logdir, flush_secs=flush_secs)\n",
+    "  File \"C:\\Users\\Taiyo\\.conda\\envs\\SMACV2\\lib\\site-packages\\tensorboard_logger\\tensorboard_logger.py\", line 51, in __init__\n    self._writer = open(filename, 'wb')\n",
+    "FileNotFoundError: [Errno 2] No such file or directory: 'C:\\\\Users\\\\Taiyo\\\\Desktop\\\\SMAC V2\\\\pymarl3\\\\results\\\\tb_logs\\\\sc2_v2_10gen_protoss-obs_aid=1-obs_act=0-conic_fov=0\\\\algo=qmix-agent=n_rnn\\\\env_n=4\\\\mixer=qmix\\\\rnn_dim=64-2bs=5000_128-tdlambda=0.6-epdec_0.05=100k\\\\qmix__2025-01-06_00-23-53\\\\events.out.tfevents.1736094233.Taiyopen'\n"
+  ],
+  "heartbeat": "2025-01-05T16:23:53.621785",
+  "host": {
+    "ENV": {},
+    "cpu": "AMD Ryzen 7 5700X3D 8-Core Processor",
+    "gpus": {
+      "driver_version": "560.94",
+      "gpus": [
+        {
+          "model": "NVIDIA GeForce RTX 4080 SUPER",
+          "persistence_mode": false,
+          "total_memory": 16376
+        }
+      ]
+    },
+    "hostname": "Taiyopen",
+    "os": [
+      "Windows",
+      "Windows-10-10.0.22631-SP0"
+    ],
+    "python_version": "3.10.16"
+  },
+  "meta": {
+    "command": "my_main",
+    "config_updates": {
+      "batch_size": 128,
+      "batch_size_run": 4,
+      "buffer_size": 5000,
+      "epsilon_anneal_time": 100000,
+      "obs_agent_id": true,
+      "obs_last_action": false,
+      "runner": "parallel",
+      "t_max": 4050000,
+      "td_lambda": 0.6,
+      "use_tensorboard": true
+    },
+    "named_configs": [],
+    "options": {
+      "--beat-interval": null,
+      "--capture": null,
+      "--comment": null,
+      "--debug": false,
+      "--enforce_clean": false,
+      "--file_storage": null,
+      "--force": false,
+      "--help": false,
+      "--id": null,
+      "--loglevel": null,
+      "--mongo_db": null,
+      "--name": null,
+      "--pdb": false,
+      "--print-config": false,
+      "--priority": null,
+      "--queue": false,
+      "--s3": null,
+      "--sql": null,
+      "--tiny_db": null,
+      "--unobserved": false,
+      "COMMAND": null,
+      "UPDATE": [
+        "obs_agent_id=True",
+        "obs_last_action=False",
+        "runner=parallel",
+        "batch_size_run=4",
+        "buffer_size=5000",
+        "t_max=4050000",
+        "epsilon_anneal_time=100000",
+        "batch_size=128",
+        "td_lambda=0.6",
+        "use_tensorboard=True"
+      ],
+      "help": false,
+      "with": true
+    }
+  },
+  "resources": [],
+  "result": null,
+  "start_time": "2025-01-05T16:23:53.552221",
+  "status": "FAILED",
+  "stop_time": "2025-01-05T16:23:53.623909"
+}

results/sacred/10gen_protoss/qmix/6/config.json

@@ -0,0 +1,130 @@
+{
+  "action_selector": "epsilon_greedy",
+  "agent": "n_rnn",
+  "agent_output_type": "q",
+  "asn_hidden_size": 32,
+  "batch_size": 128,
+  "batch_size_run": 4,
+  "buffer_cpu_only": true,
+  "buffer_size": 5000,
+  "checkpoint_path": "",
+  "cpu_inference": true,
+  "critic_lr": 0.0005,
+  "double_q": true,
+  "enable_parallel_computing": false,
+  "env": "sc2_v2",
+  "env_args": {
+    "capability_config": {
+      "n_units": 5,
+      "start_positions": {
+        "dist_type": "surrounded_and_reflect",
+        "map_x": 32,
+        "map_y": 32,
+        "n_enemies": 5,
+        "p": 0.5
+      },
+      "team_gen": {
+        "dist_type": "weighted_teams",
+        "exception_unit_types": [
+          "colossus"
+        ],
+        "observe": true,
+        "unit_types": [
+          "stalker",
+          "zealot",
+          "colossus"
+        ],
+        "weights": [
+          0.45,
+          0.45,
+          0.1
+        ]
+      }
+    },
+    "change_fov_with_move": false,
+    "conic_fov": false,
+    "continuing_episode": false,
+    "debug": false,
+    "difficulty": "7",
+    "fully_observable": false,
+    "game_version": null,
+    "heuristic_ai": false,
+    "heuristic_rest": false,
+    "kill_unit_step_mul": 2,
+    "map_name": "10gen_protoss",
+    "move_amount": 2,
+    "num_fov_actions": 12,
+    "obs_all_health": true,
+    "obs_instead_of_state": false,
+    "obs_last_action": false,
+    "obs_own_health": true,
+    "obs_own_pos": true,
+    "obs_pathing_grid": false,
+    "obs_starcraft": true,
+    "obs_terrain_height": false,
+    "obs_timestep_number": false,
+    "replay_dir": "",
+    "replay_prefix": "",
+    "reward_death_value": 10,
+    "reward_defeat": 0,
+    "reward_negative_scale": 0.5,
+    "reward_only_positive": true,
+    "reward_scale": true,
+    "reward_scale_rate": 20,
+    "reward_sparse": false,
+    "reward_win": 200,
+    "seed": null,
+    "state_last_action": true,
+    "state_timestep_number": false,
+    "step_mul": 8
+  },
+  "epsilon_anneal_time": 100000,
+  "epsilon_finish": 0.05,
+  "epsilon_start": 1.0,
+  "evaluate": false,
+  "gain": 0.01,
+  "gamma": 0.99,
+  "grad_norm_clip": 10,
+  "hypernet_embed": 64,
+  "label": "default_label",
+  "learner": "nq_learner",
+  "learner_log_interval": 10000,
+  "load_step": 0,
+  "local_results_path": "results",
+  "log_interval": 10000,
+  "lr": 0.001,
+  "mac": "n_mac",
+  "mixer": "qmix",
+  "mixing_embed_dim": 32,
+  "name": "qmix",
+  "obs_agent_id": true,
+  "obs_last_action": false,
+  "optim_alpha": 0.99,
+  "optim_eps": 1e-05,
+  "optimizer": "adam",
+  "per_alpha": 0.6,
+  "per_beta": 0.4,
+  "q_lambda": false,
+  "repeat_id": 1,
+  "return_priority": false,
+  "rnn_hidden_dim": 64,
+  "run": "default",
+  "runner": "parallel",
+  "runner_log_interval": 10000,
+  "save_model": true,
+  "save_model_interval": 2000000,
+  "save_replay": false,
+  "seed": 112463159,
+  "t_max": 4050000,
+  "target_update_interval": 200,
+  "td_lambda": 0.6,
+  "test_greedy": true,
+  "test_interval": 10000,
+  "test_nepisode": 32,
+  "thread_num": 4,
+  "use_cuda": true,
+  "use_layer_norm": false,
+  "use_orthogonal": false,
+  "use_per": false,
+  "use_tensorboard": false
+}

results/sacred/10gen_protoss/qmix/6/metrics.json

@@ -0,0 +1 @@
+{}

results/sacred/10gen_protoss/qmix/6/run.json

@@ -0,0 +1,116 @@
+{
+  "artifacts": [],
+  "command": "my_main",
+  "experiment": {
+    "base_dir": "C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src",
+    "dependencies": [
+      "numpy==1.23.1",
+      "PyYAML==6.0.2",
+      "sacred==0.8.7",
+      "torch==1.13.1+cu117"
+    ],
+    "mainfile": "main.py",
+    "name": "pymarl",
+    "repositories": [
+      {
+        "commit": "44bb188185fd0292a1a306c86396027efb44224d",
+        "dirty": true,
+        "url": "https://github.com/tjuHaoXiaotian/pymarl3.git"
+      },
+      {
+        "commit": "44bb188185fd0292a1a306c86396027efb44224d",
+        "dirty": true,
+        "url": "https://github.com/tjuHaoXiaotian/pymarl3.git"
+      }
+    ],
+    "sources": [
+      [
+        "main.py",
+        "_sources\\main_654daaa6534bcee62784d639ea63e51d.py"
+      ],
+      [
+        "utils\\logging.py",
+        "_sources\\logging_f71df6d788e929fac28afdf951d63d54.py"
+      ]
+    ]
+  },
+  "heartbeat": "2025-01-05T19:43:03.753377",
+  "host": {
+    "ENV": {},
+    "cpu": "AMD Ryzen 7 5700X3D 8-Core Processor",
+    "gpus": {
+      "driver_version": "560.94",
+      "gpus": [
+        {
+          "model": "NVIDIA GeForce RTX 4080 SUPER",
+          "persistence_mode": false,
+          "total_memory": 16376
+        }
+      ]
+    },
+    "hostname": "Taiyopen",
+    "os": [
+      "Windows",
+      "Windows-10-10.0.22631-SP0"
+    ],
+    "python_version": "3.10.16"
+  },
+  "meta": {
+    "command": "my_main",
+    "config_updates": {
+      "batch_size": 128,
+      "batch_size_run": 4,
+      "buffer_size": 5000,
+      "epsilon_anneal_time": 100000,
+      "obs_agent_id": true,
+      "obs_last_action": false,
+      "runner": "parallel",
+      "t_max": 4050000,
+      "td_lambda": 0.6,
+      "use_tensorboard": false
+    },
+    "named_configs": [],
+    "options": {
+      "--beat-interval": null,
+      "--capture": null,
+      "--comment": null,
+      "--debug": false,
+      "--enforce_clean": false,
+      "--file_storage": null,
+      "--force": false,
+      "--help": false,
+      "--id": null,
+      "--loglevel": null,
+      "--mongo_db": null,
+      "--name": null,
+      "--pdb": false,
+      "--print-config": false,
+      "--priority": null,
+      "--queue": false,
+      "--s3": null,
+      "--sql": null,
+      "--tiny_db": null,
+      "--unobserved": false,
+      "COMMAND": null,
+      "UPDATE": [
+        "obs_agent_id=True",
+        "obs_last_action=False",
+        "runner=parallel",
+        "batch_size_run=4",
+        "buffer_size=5000",
+        "t_max=4050000",
+        "epsilon_anneal_time=100000",
+        "batch_size=128",
+        "td_lambda=0.6",
+        "use_tensorboard=False"
+      ],
+      "help": false,
+      "with": true
+    }
+  },
+  "resources": [],
+  "result": null,
+  "start_time": "2025-01-05T16:24:03.185725",
+  "status": "INTERRUPTED",
+  "stop_time": "2025-01-05T19:43:04.003168"
+}

results/sacred/10gen_protoss/qmix/_sources/logging_f71df6d788e929fac28afdf951d63d54.py

@@ -0,0 +1,68 @@
+from collections import defaultdict
+import logging
+import numpy as np
+import torch as th
+
+class Logger:
+    def __init__(self, console_logger):
+        self.console_logger = console_logger
+
+        self.use_tb = False
+        self.use_sacred = False
+        self.use_hdf = False
+
+        self.stats = defaultdict(lambda: [])
+
+    def setup_tb(self, directory_name):
+        # Import here so it doesn't have to be installed if you don't use it
+        from tensorboard_logger import configure, log_value
+        configure(directory_name)
+        self.tb_logger = log_value
+        self.use_tb = True
+
+    def setup_sacred(self, sacred_run_dict):
+        self.sacred_info = sacred_run_dict.info
+        self.use_sacred = True
+
+    def log_stat(self, key, value, t, to_sacred=True):
+        self.stats[key].append((t, value))
+
+        if self.use_tb:
+            self.tb_logger(key, value, t)
+
+        if self.use_sacred and to_sacred:
+            if key in self.sacred_info:
+                self.sacred_info["{}_T".format(key)].append(t)
+                self.sacred_info[key].append(value)
+            else:
+                self.sacred_info["{}_T".format(key)] = [t]
+                self.sacred_info[key] = [value]
+
+    def print_recent_stats(self):
+        log_str = "Recent Stats | t_env: {:>10} | Episode: {:>8}\n".format(*self.stats["episode"][-1])
+        i = 0
+        for (k, v) in sorted(self.stats.items()):
+            if k == "episode":
+                continue
+            i += 1
+            window = 5 if k != "epsilon" else 1
+            item = "{:.4f}".format(th.mean(th.tensor([float(x[1]) for x in self.stats[k][-window:]])))
+            log_str += "{:<25}{:>8}".format(k + ":", item)
+            log_str += "\n" if i % 4 == 0 else "\t"
+        self.console_logger.info(log_str)
+        # Reset stats to avoid accumulating logs in memory
+        self.stats = defaultdict(lambda: [])
+
+
+# set up a custom logger
+def get_logger():
+    logger = logging.getLogger()
+    logger.handlers = []
+    ch = logging.StreamHandler()
+    formatter = logging.Formatter('[%(levelname)s %(asctime)s] %(name)s %(message)s', '%H:%M:%S')
+    ch.setFormatter(formatter)
+    logger.addHandler(ch)
+    logger.setLevel('DEBUG')
+
+    return logger
+

results/sacred/10gen_protoss/qmix/_sources/main_654daaa6534bcee62784d639ea63e51d.py

@@ -0,0 +1,124 @@
+import random
+
+import numpy as np
+import os
+import collections
+from os.path import dirname, abspath, join
+from copy import deepcopy
+from sacred import Experiment, SETTINGS
+from sacred.observers import FileStorageObserver
+from sacred.utils import apply_backspaces_and_linefeeds
+import sys
+import torch as th
+from utils.logging import get_logger
+import yaml
+import collections.abc
+
+from run import REGISTRY as run_REGISTRY
+
+SETTINGS['CAPTURE_MODE'] = "no"  # set to "no" if you want to see stdout/stderr in console "fd" or "no"
+logger = get_logger()
+
+ex = Experiment("pymarl")
+ex.logger = logger
+ex.captured_out_filter = apply_backspaces_and_linefeeds
+
+results_path = join(dirname(dirname(abspath(__file__))))
+
+
+@ex.main
+def my_main(_run, _config, _log):
+    # Setting the random seed throughout the modules
+    config = config_copy(_config)
+    random.seed(config["seed"])
+    np.random.seed(config["seed"])
+    th.manual_seed(config["seed"])
+    th.cuda.manual_seed(config["seed"])
+    # th.cuda.manual_seed_all(config["seed"])
+    th.backends.cudnn.deterministic = True  # cudnn
+
+
+    config['env_args']['seed'] = config["seed"]
+
+    # run
+    run_REGISTRY[_config['run']](_run, config, _log)
+
+
+def _get_config(params, arg_name, subfolder):
+    config_name = None
+    for _i, _v in enumerate(params):
+        if _v.split("=")[0] == arg_name:
+            config_name = _v.split("=")[1]
+            del params[_i]
+            break
+
+    if config_name is not None:
+        with open(os.path.join(os.path.dirname(__file__), "config", subfolder, "{}.yaml".format(config_name)),
+                  "r") as f:
+            try:
+                config_dict = yaml.load(f, Loader=yaml.SafeLoader)
+            except yaml.YAMLError as exc:
+                assert False, "{}.yaml error: {}".format(config_name, exc)
+        return config_dict
+
+
+def recursive_dict_update(d, u):
+    for k, v in u.items():
+        if isinstance(v, collections.abc.Mapping):
+            d[k] = recursive_dict_update(d.get(k, {}), v)
+        else:
+            d[k] = v
+    return d
+
+
+def config_copy(config):
+    if isinstance(config, dict):
+        return {k: config_copy(v) for k, v in config.items()}
+    elif isinstance(config, list):
+        return [config_copy(v) for v in config]
+    else:
+        return deepcopy(config)
+
+
+def parse_command(params, key, default):
+    result = default
+    for _i, _v in enumerate(params):
+        if _v.split("=")[0].strip() == key:
+            result = _v[_v.index('=') + 1:].strip()
+            break
+    return result
+
+
+if __name__ == '__main__':
+    params = deepcopy(sys.argv)
+
+    # Get the defaults from default.yaml
+    with open(os.path.join(os.path.dirname(__file__), "config", "default.yaml"), "r") as f:
+        try:
+            config_dict = yaml.load(f, Loader=yaml.SafeLoader)
+        except yaml.YAMLError as exc:
+            assert False, "default.yaml error: {}".format(exc)
+
+    # Load algorithm and env base configs
+    env_config = _get_config(params, "--env-config", "envs")
+    alg_config = _get_config(params, "--config", "algs")
+    # config_dict = {**config_dict, **env_config, **alg_config}
+    config_dict = recursive_dict_update(config_dict, env_config)
+    config_dict = recursive_dict_update(config_dict, alg_config)
+
+    # now add all the config to sacred
+    ex.add_config(config_dict)
+
+    # Save to disk by default for sacred
+    map_name = parse_command(params, "env_args.map_name", config_dict['env_args']['map_name'])
+    algo_name = parse_command(params, "name", config_dict['name'])
+    local_results_path = parse_command(params, "local_results_path", config_dict['local_results_path'])
+    file_obs_path = join(results_path, local_results_path, "sacred", map_name, algo_name)
+
+    logger.info("Saving to FileStorageObserver in {}.".format(file_obs_path))
+    ex.observers.append(FileStorageObserver.create(file_obs_path))
+
+    ex.run_commandline(params)
+
+    # flush
+    sys.stdout.flush()

results/sacred/5m_vs_6m/feudal/1/config.json

@@ -0,0 +1,97 @@
+{
+  "action_selector": "epsilon_greedy",
+  "agent": "feudal",
+  "agent_output_type": "q",
+  "asn_hidden_size": 32,
+  "batch_size": 128,
+  "batch_size_run": 4,
+  "buffer_cpu_only": true,
+  "buffer_size": 5000,
+  "c": 1,
+  "checkpoint_path": "",
+  "cpu_inference": true,
+  "critic_hidden_dim": 64,
+  "critic_lr": 0.0005,
+  "double_q": true,
+  "embedding_dim": 16,
+  "enable_parallel_computing": false,
+  "env": "sc2",
+  "env_args": {
+    "continuing_episode": false,
+    "debug": false,
+    "difficulty": "7",
+    "game_version": null,
+    "heuristic_ai": false,
+    "heuristic_rest": false,
+    "map_name": "5m_vs_6m",
+    "move_amount": 2,
+    "obs_all_health": true,
+    "obs_instead_of_state": false,
+    "obs_last_action": false,
+    "obs_own_health": true,
+    "obs_pathing_grid": false,
+    "obs_terrain_height": false,
+    "obs_timestep_number": false,
+    "replay_dir": "",
+    "replay_prefix": "",
+    "reward_death_value": 10,
+    "reward_defeat": 0,
+    "reward_negative_scale": 0.5,
+    "reward_only_positive": true,
+    "reward_scale": true,
+    "reward_scale_rate": 20,
+    "reward_sparse": false,
+    "reward_win": 200,
+    "seed": null,
+    "state_last_action": true,
+    "state_timestep_number": false,
+    "step_mul": 8
+  },
+  "epsilon_anneal_time": 100000,
+  "epsilon_finish": 0.05,
+  "epsilon_start": 1.0,
+  "evaluate": false,
+  "gamma": 0.99,
+  "goal_dim": 16,
+  "grad_norm_clip": 10,
+  "hypernet_embed": 64,
+  "hypernet_layers": 2,
+  "intrinsic_rewards_alpha": 0.001,
+  "label": "default_label",
+  "learner": "feudal_learner",
+  "learner_log_interval": 10000,
+  "load_step": 0,
+  "local_results_path": "results",
+  "log_interval": 10000,
+  "lr": 0.0005,
+  "mac": "feudal_mac",
+  "manager_hidden_dim": 64,
+  "mixer": "qmix",
+  "mixing_embed_dim": 32,
+  "name": "feudal",
+  "obs_agent_id": true,
+  "obs_last_action": true,
+  "optim_alpha": 0.99,
+  "optim_eps": 1e-05,
+  "repeat_id": 1,
+  "rnn_hidden_dim": 64,
+  "run": "default",
+  "runner": "parallel",
+  "runner_log_interval": 10000,
+  "save_model": true,
+  "save_model_interval": 2000000,
+  "save_replay": false,
+  "seed": 707858439,
+  "state_dim": 80,
+  "t_max": 100000,
+  "target_update_interval": 200,
+  "td_lambda": 0.6,
+  "test_greedy": true,
+  "test_interval": 10000,
+  "test_nepisode": 32,
+  "thread_num": 4,
+  "use_cuda": true,
+  "use_tensorboard": true,
+  "vf_coef": 0.5,
+  "worker_hidden_dim": 64
+}

results/sacred/5m_vs_6m/feudal/1/metrics.json

@@ -0,0 +1 @@
+{}

results/sacred/5m_vs_6m/feudal/1/run.json

@@ -0,0 +1,127 @@
+{
+  "artifacts": [],
+  "command": "my_main",
+  "experiment": {
+    "base_dir": "C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src",
+    "dependencies": [
+      "numpy==1.23.1",
+      "PyYAML==6.0.2",
+      "sacred==0.8.7",
+      "torch==1.13.1+cu117"
+    ],
+    "mainfile": "main.py",
+    "name": "pymarl",
+    "repositories": [
+      {
+        "commit": "44bb188185fd0292a1a306c86396027efb44224d",
+        "dirty": true,
+        "url": "https://github.com/tjuHaoXiaotian/pymarl3.git"
+      },
+      {
+        "commit": "44bb188185fd0292a1a306c86396027efb44224d",
+        "dirty": true,
+        "url": "https://github.com/tjuHaoXiaotian/pymarl3.git"
+      }
+    ],
+    "sources": [
+      [
+        "main.py",
+        "_sources\\main_888918ff84cf3bc1e6e9dcc8919870b2.py"
+      ],
+      [
+        "utils\\logging.py",
+        "_sources\\logging_f71df6d788e929fac28afdf951d63d54.py"
+      ]
+    ]
+  },
+  "fail_trace": [
+    "Traceback (most recent call last):\n",
+    "  File \"C:\\Users\\Taiyo\\.conda\\envs\\SMACV2\\lib\\site-packages\\sacred\\config\\captured_function.py\", line 42, in captured_function\n    result = wrapped(*args, **kwargs)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\main.py\", line 44, in my_main\n    run_REGISTRY[_config['run']](_run, config, _log)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\run\\run.py\", line 128, in run\n    run_sequential(args=args, logger=logger)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\run\\run.py\", line 209, in run_sequential\n    learner = le_REGISTRY[args.learner](mac, buffer.scheme, logger, args)\n",
+    "  File \"C:\\Users\\Taiyo\\Desktop\\SMAC V2\\pymarl3\\src\\learners\\FeUdal_learner.py\", line 45, in __init__\n    self.critic = FeudalCritic(args.state_dim, args)\n",
+    "NameError: name 'FeudalCritic' is not defined\n"
+  ],
+  "heartbeat": "2024-12-29T18:43:24.691551",
+  "host": {
+    "ENV": {},
+    "cpu": "AMD Ryzen 7 5700X3D 8-Core Processor",
+    "gpus": {
+      "driver_version": "560.94",
+      "gpus": [
+        {
+          "model": "NVIDIA GeForce RTX 4080 SUPER",
+          "persistence_mode": false,
+          "total_memory": 16376
+        }
+      ]
+    },
+    "hostname": "Taiyopen",
+    "os": [
+      "Windows",
+      "Windows-10-10.0.22631-SP0"
+    ],
+    "python_version": "3.10.16"
+  },
+  "meta": {
+    "command": "my_main",
+    "config_updates": {
+      "batch_size": 128,
+      "batch_size_run": 4,
+      "buffer_size": 5000,
+      "env_args": {
+        "map_name": "5m_vs_6m"
+      },
+      "epsilon_anneal_time": 100000,
+      "obs_agent_id": true,
+      "obs_last_action": true,
+      "runner": "parallel",
+      "t_max": 100000,
+      "td_lambda": 0.6
+    },
+    "named_configs": [],
+    "options": {
+      "--beat-interval": null,
+      "--capture": null,
+      "--comment": null,
+      "--debug": false,
+      "--enforce_clean": false,
+      "--file_storage": null,
+      "--force": false,
+      "--help": false,
+      "--id": null,
+      "--loglevel": null,
+      "--mongo_db": null,
+      "--name": null,
+      "--pdb": false,
+      "--print-config": false,
+      "--priority": null,
+      "--queue": false,
+      "--s3": null,
+      "--sql": null,
+      "--tiny_db": null,
+      "--unobserved": false,
+      "COMMAND": null,
+      "UPDATE": [
+        "env_args.map_name=5m_vs_6m",
+        "obs_agent_id=True",
+        "obs_last_action=True",
+        "runner=parallel",
+        "batch_size_run=4",
+        "buffer_size=5000",
+        "t_max=100000",
+        "epsilon_anneal_time=100000",
+        "batch_size=128",
+        "td_lambda=0.6"
+      ],
+      "help": false,
+      "with": true
+    }
+  },
+  "resources": [],
+  "result": null,
+  "start_time": "2024-12-29T18:43:20.056773",
+  "status": "FAILED",
+  "stop_time": "2024-12-29T18:43:24.693562"
+}

results/sacred/5m_vs_6m/feudal/10/config.json

@@ -0,0 +1,97 @@
+{
+  "action_selector": "epsilon_greedy",
+  "agent": "feudal",
+  "agent_output_type": "q",
+  "asn_hidden_size": 32,
+  "batch_size": 128,
+  "batch_size_run": 4,
+  "buffer_cpu_only": true,
+  "buffer_size": 5000,
+  "c": 1,
+  "checkpoint_path": "",
+  "cpu_inference": true,
+  "critic_hidden_dim": 64,
+  "critic_lr": 0.0005,
+  "double_q": true,
+  "embedding_dim": 16,
+  "enable_parallel_computing": false,
+  "env": "sc2",
+  "env_args": {
+    "continuing_episode": false,
+    "debug": false,
+    "difficulty": "7",
+    "game_version": null,
+    "heuristic_ai": false,
+    "heuristic_rest": false,
+    "map_name": "5m_vs_6m",
+    "move_amount": 2,
+    "obs_all_health": true,
+    "obs_instead_of_state": false,
+    "obs_last_action": false,
+    "obs_own_health": true,
+    "obs_pathing_grid": false,
+    "obs_terrain_height": false,
+    "obs_timestep_number": false,
+    "replay_dir": "",
+    "replay_prefix": "",
+    "reward_death_value": 10,
+    "reward_defeat": 0,
+    "reward_negative_scale": 0.5,
+    "reward_only_positive": true,
+    "reward_scale": true,
+    "reward_scale_rate": 20,
+    "reward_sparse": false,
+    "reward_win": 200,
+    "seed": null,
+    "state_last_action": true,
+    "state_timestep_number": false,
+    "step_mul": 8
+  },
+  "epsilon_anneal_time": 100000,
+  "epsilon_finish": 0.05,
+  "epsilon_start": 1.0,
+  "evaluate": false,
+  "gamma": 0.99,
+  "goal_dim": 16,
+  "grad_norm_clip": 10,
+  "hypernet_embed": 64,
+  "hypernet_layers": 2,
+  "intrinsic_rewards_alpha": 0.001,
+  "label": "default_label",
+  "learner": "feudal_learner",
+  "learner_log_interval": 10000,
+  "load_step": 0,
+  "local_results_path": "results",
+  "log_interval": 10000,
+  "lr": 0.0005,
+  "mac": "feudal_mac",
+  "manager_hidden_dim": 64,
+  "mixer": "qmix",
+  "mixing_embed_dim": 32,
+  "name": "feudal",
+  "obs_agent_id": true,
+  "obs_last_action": true,
+  "optim_alpha": 0.99,
+  "optim_eps": 1e-05,
+  "repeat_id": 1,
+  "rnn_hidden_dim": 64,
+  "run": "default",
+  "runner": "parallel",
+  "runner_log_interval": 10000,
+  "save_model": true,
+  "save_model_interval": 2000000,
+  "save_replay": false,
+  "seed": 507787181,
+  "state_dim": 80,
+  "t_max": 100000,
+  "target_update_interval": 200,
+  "td_lambda": 0.6,
+  "test_greedy": true,
+  "test_interval": 10000,
+  "test_nepisode": 32,
+  "thread_num": 4,
+  "use_cuda": true,
+  "use_tensorboard": true,
+  "vf_coef": 0.5,
+  "worker_hidden_dim": 64
+}