diff --git a/3rdParty/itk-io.LICENSE.txt b/3rdParty/itk-io.LICENSE.txt
new file mode 100644
index 00000000000..d6456956733
--- /dev/null
+++ b/3rdParty/itk-io.LICENSE.txt
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/3rdParty/kaleido.LICENSE.txt b/3rdParty/kaleido.LICENSE.txt
new file mode 100644
index 00000000000..904dd47bba6
--- /dev/null
+++ b/3rdParty/kaleido.LICENSE.txt
@@ -0,0 +1,267 @@
+All code developed in this repository is released under the MIT license (1).
+
+The Kaleido executable includes Chromium, which is released under the 3-clause BSD license (2).
+
+Chromium includes a wide range of third-party dependencies with varying licenses.
+See the CREDITS.html file distributed with this package for license details
+of these dependencies.
+
+The CREDITS.html is also available at https://github.com/plotly/Kaleido/blob/master/repos/CREDITS.html
+
+The Kaleido python PyPI package vendors MathJax, which is released under the terms of the Apache License (3)
+
+(1) ----
+The MIT License (MIT)
+
+Copyright (c) 2020 Plotly, Inc
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+(2) ----
+// Copyright 2015 The Chromium Authors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+(3) ----
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
diff --git a/3rdParty/monai.LICENSE.txt b/3rdParty/monai.LICENSE.txt
new file mode 100644
index 00000000000..261eeb9e9f8
--- /dev/null
+++ b/3rdParty/monai.LICENSE.txt
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/3rdParty/pandas.LICENSE.txt b/3rdParty/pandas.LICENSE.txt
new file mode 100644
index 00000000000..a0cc369f725
--- /dev/null
+++ b/3rdParty/pandas.LICENSE.txt
@@ -0,0 +1,31 @@
+BSD 3-Clause License
+
+Copyright (c) 2008-2011, AQR Capital Management, LLC, Lambda Foundry, Inc. and PyData Development Team
+All rights reserved.
+
+Copyright (c) 2011-2021, Open source contributors.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/3rdParty/plotly.LICENSE.txt b/3rdParty/plotly.LICENSE.txt
new file mode 100644
index 00000000000..98b37b6f421
--- /dev/null
+++ b/3rdParty/plotly.LICENSE.txt
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016-2018 Plotly, Inc
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/docs/examples/hello_pt.rst b/docs/examples/hello_pt.rst
index 8899eb03387..93ba9015404 100644
--- a/docs/examples/hello_pt.rst
+++ b/docs/examples/hello_pt.rst
@@ -4,7 +4,7 @@ Quickstart (PyTorch)
Before You Start
----------------
-Feel free to refer to the official :doc:`documentation <../programming_guide>` at any point to learn more about the specifics of `NVIDIA FLARE `_.
+Feel free to refer to the :doc:`detailed documentation <../programming_guide>` at any point to learn more about the specifics of `NVIDIA FLARE `_.
Make sure you have an environment with NVIDIA FLARE installed. You can follow the
:doc:`installation <../installation>` guide on the general concept of Python virtual environment (the recommended environment) and how to
diff --git a/docs/faq.rst b/docs/faq.rst
index 382f1a75ede..7248c791f4e 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -387,3 +387,5 @@ Known issues
large to be uploaded and that causes timeout.
#. Please don't start a new training run or start a new app before the previous application is fully stopped. Users
can do ``abort client`` and ``abort server`` before ``start_app`` for the new run.
+#. After calling ``shutdown client`` for a client running multi GPUs, a process (sub_worker_process) may remain. The
+ work around for this is to run ``abort client`` before the ``shutdown`` command.
diff --git a/docs/programming_guide/controllers.rst b/docs/programming_guide/controllers.rst
index 3f8d5272253..5b26c4785e2 100644
--- a/docs/programming_guide/controllers.rst
+++ b/docs/programming_guide/controllers.rst
@@ -9,15 +9,12 @@ Controller/Worker Interactions
NVIDIA FLARE 2.0's collaborative computing is achieved through the Controller/Worker interactions. The following diagram
shows how the Controller and Worker interact.
-.. image:: ../resources/Controller_worker.png
+.. image:: ../resources/Controller.png
:height: 300px
The Controller is a python object that controls or coordinates the Workers to get a job done. The controller is run on
the FL server (highlighted on the right).
-.. image:: ../resources/Controller.png
- :height: 300px
-
A Worker is capable of performing tasks. Workers run on FL clients.
In its control logic, the Controller assigns tasks to Workers and processes task results from the Workers.
diff --git a/docs/resources/Controller_worker.png b/docs/resources/Controller_worker.png
deleted file mode 100644
index 39f7b695e08..00000000000
Binary files a/docs/resources/Controller_worker.png and /dev/null differ
diff --git a/examples/README.md b/examples/README.md
index adf4dea7439..554a3e6c4cd 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -17,6 +17,8 @@ The provided examples cover different aspects of [NVIDIA FLARE](https://nvidia.g
### 1.2 Deep Learning
* [Hello PyTorch](./hello-pt/README.md)
* Example using [NVIDIA FLARE](https://nvidia.github.io/NVFlare) an image classifier using [FedAvg]([FedAvg](https://arxiv.org/abs/1602.05629)) and [PyTorch](https://pytorch.org/) as the deep learning training framework.
+* [Hello PyTorch with TensorBoard](./hello-pt-tb/README.md)
+ * Example building upon [Hello PyTorch](./hello-pt/README.md) showcasing the [TensorBoard](https://tensorflow.org/tensorboard) streaming capability from the clients to the server.
* [Hello TensorFlow](./hello-tf2/README.md)
* Example of using [NVIDIA FLARE](https://nvidia.github.io/NVFlare) an image classifier using [FedAvg]([FedAvg](https://arxiv.org/abs/1602.05629)) and [TensorFlow](https://tensorflow.org/) as the deep learning training framework.
@@ -30,4 +32,6 @@ The provided examples cover different aspects of [NVIDIA FLARE](https://nvidia.g
* [Federated Learning with Differential Privacy for BraTS18 segmentation](./brats18/README.md)
* Illustrates the use of differential privacy for training brain tumor segmentation models using federated learning.
* [Federated Learning for Prostate Segmentation from Multi-source Data](./prostate/README.md)
- * Example of training a multi-institutional prostate segmentation model using [FedAvg](https://arxiv.org/abs/1602.05629), [FedProx](https://arxiv.org/abs/1812.06127), and [Ditto](https://arxiv.org/abs/2012.04221).
+ * Example of training a multi-institutional prostate segmentation model using [FedAvg](https://arxiv.org/abs/1602.05629), [FedProx](https://arxiv.org/abs/1812.06127), and [Ditto](https://arxiv.org/abs/2012.04221).
+* [Federated Analysis](./federated_analysis/README.md)
+ * Example of gathering local data summary statistics to compute the global dataset statistics.
diff --git a/examples/cifar10/figs/plot_tensorboard_events.py b/examples/cifar10/figs/plot_tensorboard_events.py
index ee5b331e763..1d35d0fffe9 100644
--- a/examples/cifar10/figs/plot_tensorboard_events.py
+++ b/examples/cifar10/figs/plot_tensorboard_events.py
@@ -76,7 +76,7 @@ def main():
data = {"Config": [], "Step": [], "Accuracy": []}
if add_cross_site_val:
- xsite_keys = ["SRV_server", "SRV_server_best"]
+ xsite_keys = ["SRV_FL_global_model.pt", "SRV_best_FL_global_model.pt"]
xsite_data = {"Config": []}
for k in xsite_keys:
xsite_data.update({k: []})
@@ -94,7 +94,7 @@ def main():
if add_cross_site_val:
xsite_file = glob.glob(
- os.path.join(server_results_root, exp["run"] + "/**/cross_site_val.json"), recursive=True
+ os.path.join(server_results_root, exp["run"] + "/**/cross_val_results.json"), recursive=True
)
assert len(xsite_file) == 1, "No unique x-site file found!"
with open(xsite_file[0], "r") as f:
diff --git a/examples/federated_analysis/.gitignore b/examples/federated_analysis/.gitignore
new file mode 100644
index 00000000000..e8e7d950386
--- /dev/null
+++ b/examples/federated_analysis/.gitignore
@@ -0,0 +1,23 @@
+# ide
+.idea/
+.ipynb_checkpoints/
+
+# nvflare artifacts
+log.txt
+client_token.txt
+*.fl
+audit.log
+transfer
+workspaces
+
+# python
+__pycache__
+.pyc
+
+# virtual environments
+fedanalysis
+
+# data
+dataset
+dataset*
+*results*
diff --git a/examples/federated_analysis/README.md b/examples/federated_analysis/README.md
new file mode 100644
index 00000000000..6d67062d96e
--- /dev/null
+++ b/examples/federated_analysis/README.md
@@ -0,0 +1,94 @@
+# Federated Analysis with NVIDIA FLARE
+
+## (Optional) 0. Set up a virtual environment
+```
+python3 -m pip install --user --upgrade pip
+python3 -m pip install --user virtualenv
+```
+(If needed) make all shell scripts executable using
+```
+find . -name ".sh" -exec chmod +x {} \;
+```
+initialize virtual environment.
+```
+source ./virtualenv/set_env.sh
+```
+install required packages.
+```
+pip install --upgrade pip
+pip install -r ./virtualenv/requirements.txt
+```
+
+## 1. Download the example data
+
+As an example, you can download the dataset from the ["COVID-19 Radiography Database"](https://www.kaggle.com/tawsifurrahman/covid19-radiography-database).
+Download the `archive.zip` and extract to `./data/.`.
+
+Next, create the data lists simulating different clients with varying amounts and types of images.
+The downloaded archive contains subfolders for four different classes: `COVID`, `Lung_Opacity`, `Normal`, and `Viral Pneumonia`.
+Here we assume each class of image corresponds to a different sites.
+```
+python3 data/prepare_data.py --input_dir ./data
+```
+
+With this ratio setting, site-3 will have the largest number of images. You should see the following output
+```
+Created 4 data lists for ['COVID', 'Lung_Opacity', 'Normal', 'Viral Pneumonia'].
+Saved 3616 entries at ./data/site-1_COVID.json
+Saved 6012 entries at ./data/site-2_Lung_Opacity.json
+Saved 10192 entries at ./data/site-3_Normal.json
+Saved 1345 entries at ./data/site-4_Viral Pneumonia.json
+```
+
+## 2. Create your POC workspace
+To run FL experiments in POC mode, create your local FL workspace the below command.
+In the following experiments, we will be using three clients. One for each data list prepared above. Press "y" when prompted.
+```
+./create_poc_workpace.sh 4
+```
+
+## 3. Compute the local and global intensity histograms
+
+First, we add the current directory path to `config_fed_client.json` files to generate the absolute path for `data_root`.
+```
+sed -i "s|PWD|${PWD}|g" configs/fed_analysis/config/config_fed_client.json
+```
+Next, we start the federated analysis by executing the [run_poc.sh](./run_poc.sh) script. This will start the federated workflow automatically, execute the tasks on the clients and gather the histograms on the server.
+
+**Note:** The [run_poc.sh](./run_poc.sh) script follows this pattern: `./run_poc.sh [n_clients] [config] [run]`
+
+FOr example, to run three clients and under `run_1`, run the following:
+```
+./run_poc.sh 4 fed_analysis 1
+```
+
+**Note:** This example uses the [k-anonymity](https://en.wikipedia.org/wiki/K-anonymity) approach to ensure that no individual patient's data is leaked to the server.
+Clients will only send intensity histogram statistics if computed on at least `k` images. The default number is set by `min_images=10` in `AnalysisExecutor`.
+
+Other default parameters of the `AnalysisExecutor` are chosen to work well with the used example data. For other datasets, the histogram parameters (`n_bins`, `range_min`, and `range_max`) might need to be adjusted.
+
+## 4. Visualize the result
+
+If successful, the computed histograms will be shown in the `run_*` folders as `histograms.html` and `histograms.svg`.
+For example, the gathered local and global histograms will look like this.
+
+![Example local and global histograms](./histograms_example.svg)
+
+## 5. Get results using the admin client
+
+In real-world FL scenarios, the researcher might not have direct access to the server machine. Hence, the researcher can use the admin client console to control the experimentation. See [here](https://nvidia.github.io/NVFlare/user_guide/admin_commands.html) for details.
+After completing the federated analysis run, you can check the histogram files have been created:
+```
+> ls server run_1
+app_server
+fl_app.txt
+histograms.html
+histograms.svg
+```
+The result can be downloaded to the admin's machine using this command:
+```
+> download_folder ../run_1
+```
+
+After download, the files will be available in the admin workspace under `transfer`.
+
diff --git a/examples/federated_analysis/configs/fed_analysis/config/config_fed_client.json b/examples/federated_analysis/configs/fed_analysis/config/config_fed_client.json
new file mode 100644
index 00000000000..a9219e0130e
--- /dev/null
+++ b/examples/federated_analysis/configs/fed_analysis/config/config_fed_client.json
@@ -0,0 +1,17 @@
+{
+ "format_version": 2,
+ "executors": [
+ {
+ "tasks": ["histogram"],
+ "executor": {
+ "path": "analysis_executor.AnalysisExecutor",
+ "args": {
+ "data_root": "PWD/data"
+ }
+ }
+ }
+ ],
+ "task_result_filters": [],
+ "task_data_filters": [],
+ "components": []
+}
diff --git a/examples/federated_analysis/configs/fed_analysis/config/config_fed_server.json b/examples/federated_analysis/configs/fed_analysis/config/config_fed_server.json
new file mode 100644
index 00000000000..a00376e46e5
--- /dev/null
+++ b/examples/federated_analysis/configs/fed_analysis/config/config_fed_server.json
@@ -0,0 +1,18 @@
+{
+ "format_version": 2,
+ "server": {
+ "heart_beat_timeout": 600
+ },
+ "task_data_filters": [],
+ "task_result_filters": [],
+ "workflows": [
+ {
+ "id": "server_workflow",
+ "path": "analysis_controller.AnalysisController",
+ "args": {
+ "min_clients": 4
+ }
+ }
+ ],
+ "components": []
+}
diff --git a/examples/federated_analysis/configs/fed_analysis/custom/analysis_controller.py b/examples/federated_analysis/configs/fed_analysis/custom/analysis_controller.py
new file mode 100644
index 00000000000..3441443e622
--- /dev/null
+++ b/examples/federated_analysis/configs/fed_analysis/custom/analysis_controller.py
@@ -0,0 +1,172 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import numpy as np
+import plotly.graph_objects as go
+from analysis_executor import SupportedTasks
+from plotly.subplots import make_subplots
+
+from nvflare.apis.client import Client
+from nvflare.apis.dxo import from_shareable
+from nvflare.apis.fl_constant import FLContextKey, ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.impl.controller import ClientTask, Controller, Task
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+
+
+class AnalysisController(Controller):
+ def __init__(self, min_clients: int = 1):
+ """Controller for federated analysis.
+
+ Args:
+ min_clients: how many statistics to gather before computing the global statisitcs.
+ """
+ super().__init__()
+ self.histograms = dict()
+ self._min_clients = min_clients
+ self.run_dir = None
+
+ def start_controller(self, fl_ctx: FLContext):
+ self.run_dir = os.path.join(fl_ctx.get_prop(FLContextKey.APP_ROOT), "..")
+
+ def stop_controller(self, fl_ctx: FLContext):
+ pass
+
+ def process_result_of_unknown_task(
+ self,
+ client: Client,
+ task_name: str,
+ client_task_id: str,
+ result: Shareable,
+ fl_ctx: FLContext,
+ ):
+ self.log_warning(fl_ctx, f"Unknown task: {task_name} from client {client.name}.")
+
+ def control_flow(self, abort_signal: Signal, fl_ctx: FLContext):
+ self.log_info(fl_ctx, "Analysis control flow started.")
+ if abort_signal.triggered:
+ return
+ task = Task(name=SupportedTasks.HISTOGRAM, data=Shareable(), result_received_cb=self._process_result_histogram)
+ self.broadcast_and_wait(
+ task=task,
+ min_responses=self._min_clients,
+ fl_ctx=fl_ctx,
+ wait_time_after_min_received=1,
+ abort_signal=abort_signal,
+ )
+ if abort_signal.triggered:
+ return
+
+ # Visualize the histograms
+ self._create_tb_histograms(fl_ctx)
+
+ self.log_info(fl_ctx, "Analysis control flow finished.")
+
+ def _process_result_histogram(self, client_task: ClientTask, fl_ctx: FLContext):
+ task_name = client_task.task.name
+ client_name = client_task.client.name
+ self.log_info(fl_ctx, f"Processing {task_name} result from client {client_name}")
+ result = client_task.result
+ rc = result.get_return_code()
+
+ if rc == ReturnCode.OK:
+ dxo = from_shareable(result)
+ data_stat_dict = dxo.data
+ self.log_info(fl_ctx, f"Received result entries {data_stat_dict.keys()}")
+ self.histograms.update({client_name: data_stat_dict})
+ if "histogram" in data_stat_dict.keys():
+ self.log_info(fl_ctx, f"Client {client_name} finished {task_name} returned histogram.")
+ else:
+ self.log_info(fl_ctx, f"Client {client_name} finished {task_name} but return no histogram.")
+ else:
+ self.log_error(fl_ctx, f"Ignore the client train result. {task_name} tasked returned error code: {rc}")
+
+ # Cleanup task result
+ client_task.result = None
+
+ def _create_tb_histograms(self, fl_ctx: FLContext):
+ global_histogram = None
+ global_bin_edges = None
+ global_n_images = 0
+ global_n_included_images = 0
+
+ n_clients = len(self.histograms.keys())
+ # compute global histogram and plot local ones.
+ if n_clients == 0:
+ self.log_warning(fl_ctx, "There are no histograms!")
+ return
+
+ fig = make_subplots(rows=2, cols=n_clients)
+ n_plots = 0
+
+ i = 0
+ for client_name, _histo in self.histograms.items():
+ global_n_images += _histo.get("n_images", 0)
+ global_n_included_images += _histo.get("n_included_images", 0)
+
+ # compute global histogram
+ if "histogram" in _histo:
+ if global_histogram is None:
+ global_histogram = _histo["histogram"]
+ global_bin_edges = _histo["bin_edges"]
+ else: # add to current histogram
+ if np.all(np.equal(_histo["bin_edges"], global_bin_edges)):
+ global_histogram += _histo["histogram"]
+ else:
+ self.log_warning(
+ fl_ctx,
+ f"bin edges don't match the initial global bin edges. "
+ f"Ignoring results from {client_name} in global histogram.",
+ )
+
+ # write local histogram to TensorBoard
+ i += 1
+ fig.add_trace(
+ go.Scatter(
+ x=_histo["bin_edges"],
+ y=_histo["histogram"],
+ name=f"{client_name} "
+ f"({_histo.get('n_included_images', 0)} of {_histo.get('n_images', 0)} images)",
+ mode="lines",
+ fill="tozeroy",
+ ),
+ row=1,
+ col=i,
+ )
+ n_plots += 1
+ if global_histogram is not None:
+ fig.add_trace(
+ go.Scatter(
+ x=global_bin_edges,
+ y=global_histogram,
+ name=f"Global ({global_n_included_images} of {global_n_images} images)",
+ mode="lines",
+ fill="tozeroy",
+ ),
+ row=2,
+ col=1,
+ )
+
+ fig.update_layout(
+ height=600,
+ width=800,
+ title_text=f"Histograms for {n_plots} of {n_clients} clients",
+ xaxis_title="Image Intensity",
+ yaxis_title="Count",
+ )
+ fig.write_html(os.path.join(self.run_dir, "histograms.html"))
+ fig.write_image(os.path.join(self.run_dir, "histograms.svg"))
diff --git a/examples/federated_analysis/configs/fed_analysis/custom/analysis_executor.py b/examples/federated_analysis/configs/fed_analysis/custom/analysis_executor.py
new file mode 100644
index 00000000000..639ecdf349f
--- /dev/null
+++ b/examples/federated_analysis/configs/fed_analysis/custom/analysis_executor.py
@@ -0,0 +1,157 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import glob
+import os
+
+import numpy as np
+from monai.data import ITKReader, load_decathlon_datalist
+from monai.transforms import LoadImage
+
+from nvflare.apis.dxo import DXO, DataKind
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_constant import ReservedKey, ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable, make_reply
+from nvflare.apis.signal import Signal
+
+
+class SupportedTasks(object):
+ HISTOGRAM = "histogram"
+
+
+class AnalysisExecutor(Executor):
+ def __init__(
+ self,
+ data_root: str = "./data",
+ data_list_key: str = "data",
+ min_images: int = 10,
+ n_bins: int = 256,
+ range_min: float = 0.0,
+ range_max: float = 255.0, # assumes BMP/PNG/JPEGSs as input
+ ):
+ """Executor for federated analysis.
+
+ Args:
+ data_root: directory with local image data.
+ data_list_key: data list key to use.
+ min_images: minimum number of images needed to compute histogram (if less images are included in the computation, no statistics will be sent to the server).
+ n_bins: number of bins for the histogram.
+ range_min: minimum intensity value to include in the histogram.
+ range_max: maximum intensity value to include in the histogram
+
+ Returns:
+ a Shareable with the computed statistics after `execute()`
+ """
+ super().__init__()
+ self.data_list_key = data_list_key
+ self.data_root = data_root
+ self.data = None
+
+ self.n_bins = n_bins
+ self.range_min = range_min
+ self.range_max = range_max
+
+ self.data_list = None
+
+ self._min_images = min_images
+
+ self.loader = LoadImage()
+ self.loader.register(ITKReader())
+
+ def _load_data_list(self, client_name, fl_ctx: FLContext):
+ dataset_json = glob.glob(os.path.join(self.data_root, client_name + "*.json"))
+ if len(dataset_json) != 1:
+ self.log_error(
+ fl_ctx, f"No unique matching dataset list found in {self.data_root} for client {client_name}"
+ )
+ return False
+ dataset_json = dataset_json[0]
+ self.log_info(fl_ctx, f"Reading data from {dataset_json}")
+ self.data_list = load_decathlon_datalist(
+ data_list_file_path=dataset_json, data_list_key=self.data_list_key, base_dir=self.data_root
+ )
+ self.log_info(fl_ctx, f"Client {client_name} has {len(self.data_list)} images")
+ return True
+
+ def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort_signal: Signal) -> Shareable:
+ self.log_info(fl_ctx, f"Executing {task_name}")
+ try:
+ client_name = fl_ctx.get_prop(ReservedKey.CLIENT_NAME)
+ if not self._load_data_list(client_name, fl_ctx):
+ self.log_error(fl_ctx, f"Reading data list for client {client_name} failed!")
+ return make_reply(ReturnCode.ERROR)
+
+ if task_name == SupportedTasks.HISTOGRAM:
+ result_dict = self._compute_histo(fl_ctx, abort_signal)
+ if abort_signal.triggered:
+ return make_reply(ReturnCode.TASK_ABORTED)
+
+ if result_dict:
+ dxo = DXO(data_kind=DataKind.METRICS, data=result_dict)
+ return dxo.to_shareable()
+ else:
+ return make_reply(ReturnCode.EXECUTION_EXCEPTION)
+
+ else:
+ self.log_error(fl_ctx, f"{task_name} is not a supported task.")
+ return make_reply(ReturnCode.TASK_UNKNOWN)
+ except BaseException as e:
+ self.log_exception(fl_ctx, f"Task {task_name} failed. Exception: {e.__str__()}")
+ return make_reply(ReturnCode.EXECUTION_EXCEPTION)
+
+ def _compute_histo(self, fl_ctx: FLContext, abort_signal: Signal):
+ n_images = len(self.data_list)
+ n_included_images = 0
+ bin_edges = []
+
+ if n_images < self._min_images: # don't send stats if too few cases available
+ return {
+ "n_images": n_images,
+ "n_included_images": n_included_images,
+ }
+
+ histogram = np.zeros((self.n_bins,), dtype=np.int64)
+ for i, entry in enumerate(self.data_list): # TODO: use multi-processing
+ if abort_signal.triggered:
+ return None
+ file = entry.get("image")
+
+ try:
+ img, meta = self.loader(file)
+ curr_histogram, bin_edges = np.histogram(img, bins=self.n_bins, range=(self.range_min, self.range_max))
+ histogram += curr_histogram
+ n_included_images += 1
+
+ if i % 100 == 0:
+ self.log_info(fl_ctx, f"adding {i + 1} of {len(self.data_list)}: {file}")
+ except BaseException as e:
+ self.log_exception(
+ fl_ctx, f"Failed to load file {file} with exception: {e.__str__()}. Skipping this image..."
+ )
+
+ if n_included_images < self._min_images: # don't send stats if too few cases were included
+ return {
+ "n_images": n_images,
+ "n_included_images": n_included_images,
+ }
+
+ self.log_info(fl_ctx, f"Computed histogram for {n_included_images} of {n_images} images.")
+
+ return {
+ "n_images": n_images,
+ "n_included_images": n_included_images,
+ "histogram": histogram,
+ "bin_edges": bin_edges,
+ }
diff --git a/examples/federated_analysis/create_poc_workpace.sh b/examples/federated_analysis/create_poc_workpace.sh
new file mode 100755
index 00000000000..396b57beb58
--- /dev/null
+++ b/examples/federated_analysis/create_poc_workpace.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+workspace="poc_workspace"
+
+n_clients=$1
+
+if test -z "${n_clients}"
+then
+ echo "Usage: ./create_poc_workspace.sh [n_clients], e.g. ./create_poc_workspace.sh 4"
+ exit 1
+fi
+
+cur_dir=${PWD}
+
+# create POC startup kits
+cd "workspaces" || exit
+python3 -m nvflare.lighter.poc -n "${n_clients}" || exit
+# There should be $n_clients site-N folders.
+
+# move the folder
+mv "poc" ${workspace}
+echo "Created POC workspace at ./workspaces/${workspace}"
+
+cd "${cur_dir}" || exit
diff --git a/examples/federated_analysis/data/prepare_data.py b/examples/federated_analysis/data/prepare_data.py
new file mode 100644
index 00000000000..d5d764c95d1
--- /dev/null
+++ b/examples/federated_analysis/data/prepare_data.py
@@ -0,0 +1,86 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import glob
+import json
+import os
+import random
+
+SEED = 0
+
+
+def create_datasets(root, subdirs, extension, shuffle, seed):
+ random.seed(seed)
+
+ data_lists = []
+ for subdir in subdirs:
+ search_string = os.path.join(root, "**", subdir, "*" + extension)
+ data_list = glob.glob(search_string, recursive=True)
+
+ assert (
+ len(data_list) > 0
+ ), f"No images found using {search_string} for subdir '{subdir}' and extension '{extension}'!"
+
+ if shuffle:
+ random.shuffle(data_list)
+
+ data_lists.append(data_list)
+
+ return data_lists
+
+
+def save_data_list(data, data_list_file, data_root, key="data"):
+ data_list = []
+ for d in data:
+ data_list.append({"image": d.replace(data_root + os.path.sep, "")})
+
+ os.makedirs(os.path.dirname(data_list_file), exist_ok=True)
+ with open(data_list_file, "w") as f:
+ json.dump({key: data_list}, f, indent=4)
+
+ print(f"Saved {len(data_list)} entries at {data_list_file}")
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--input_dir", type=str, required=True, help="Location of image files")
+ parser.add_argument("--input_ext", type=str, default=".png", help="Search extions")
+ parser.add_argument("--output_dir", type=str, default="./data", help="Output location of data lists")
+ parser.add_argument(
+ "--subdirs",
+ type=str,
+ default="COVID,Lung_Opacity,Normal,Viral Pneumonia",
+ help="A list of subfolders to include.",
+ )
+ args = parser.parse_args()
+
+ assert "," in args.subdirs, "Expecting a comma separated list of subdirs names"
+ subdirs = [sd for sd in args.subdirs.split(",")]
+
+ data_lists = create_datasets(
+ root=args.input_dir, subdirs=subdirs, extension=args.input_ext, shuffle=True, seed=SEED
+ )
+ print(f"Created {len(data_lists)} data lists for {subdirs}.")
+
+ site_id = 1
+ for subdir, data_list in zip(subdirs, data_lists):
+ save_data_list(
+ data_list, os.path.join(args.output_dir, f"site-{site_id}_{subdir}.json"), data_root=args.input_dir
+ )
+ site_id += 1
+
+
+if __name__ == "__main__":
+ main()
diff --git a/examples/federated_analysis/histograms_example.html b/examples/federated_analysis/histograms_example.html
new file mode 100644
index 00000000000..7e91052f0f1
--- /dev/null
+++ b/examples/federated_analysis/histograms_example.html
@@ -0,0 +1,71 @@
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/examples/federated_analysis/histograms_example.svg b/examples/federated_analysis/histograms_example.svg
new file mode 100644
index 00000000000..b2dfa9db5e3
--- /dev/null
+++ b/examples/federated_analysis/histograms_example.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/examples/federated_analysis/run_fl.py b/examples/federated_analysis/run_fl.py
new file mode 100644
index 00000000000..b8490d925da
--- /dev/null
+++ b/examples/federated_analysis/run_fl.py
@@ -0,0 +1,66 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import time
+
+from nvflare.fuel.hci.client.fl_admin_api_runner import FLAdminAPIRunner
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--run_number", type=int, default=100, help="FL run number to start at.")
+ parser.add_argument("--admin_dir", type=str, default="./admin/", help="Path to admin directory.")
+ parser.add_argument("--username", type=str, default="admin@nvidia.com", help="Admin username")
+ parser.add_argument("--app", type=str, default="fed_analysis", help="App to be deployed")
+ parser.add_argument("--port", type=int, default=8003, help="The admin server port")
+ parser.add_argument("--poc", action='store_true', help="Whether admin uses POC mode.")
+ parser.add_argument("--min_clients", type=int, default=8, help="Minimum number of clients.")
+ args = parser.parse_args()
+
+ host = ""
+ port = args.port
+
+ assert os.path.isdir(args.admin_dir), f"admin directory does not exist at {args.admin_dir}"
+
+ # Set up certificate names and admin folders
+ upload_dir = os.path.join(args.admin_dir, "transfer")
+ if not os.path.isdir(upload_dir):
+ os.makedirs(upload_dir)
+ download_dir = os.path.join(args.admin_dir, "download")
+ if not os.path.isdir(download_dir):
+ os.makedirs(download_dir)
+
+ run_number = args.run_number
+
+ # Initialize the runner
+ runner = FLAdminAPIRunner(
+ host=host,
+ port=port,
+ username=args.username,
+ admin_dir=args.admin_dir,
+ poc=args.poc,
+ debug=False,
+ )
+
+ # Run
+ start = time.time()
+ runner.run(run_number, args.app, restart_all_first=False, shutdown_on_error=True, shutdown_at_end=True,
+ timeout=7200, min_clients=args.min_clients) # will time out if not completed in 2 hours
+ print("Total training time", time.time() - start)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/examples/federated_analysis/run_poc.sh b/examples/federated_analysis/run_poc.sh
new file mode 100755
index 00000000000..3a6ebb17ada
--- /dev/null
+++ b/examples/federated_analysis/run_poc.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# add current folder to PYTHONPATH
+export PYTHONPATH="${PWD}"
+echo "PYTHONPATH is ${PYTHONPATH}"
+
+algorithms_dir="${PWD}/configs"
+servername="localhost"
+workspace="workspaces/poc_workspace"
+admin_username="admin" # default admin
+site_pre="site-"
+
+n_clients=$1
+config=$2
+run=$3
+
+if test -z "${n_clients}" || test -z "${config}" || test -z "${run}"
+then
+ echo "Usage: ./run_poc.sh [n_clients] [config] [run], e.g. ./run_poc.sh 4 fed_analysis 1"
+ exit 1
+fi
+
+# start server
+echo "STARTING SERVER"
+export CUDA_VISIBLE_DEVICES=0
+./${workspace}/server/startup/start.sh ${servername} &
+sleep 10
+
+# start clients
+echo "STARTING ${n_clients} CLIENTS"
+for id in $(eval echo "{1..$n_clients}")
+do
+ export CUDA_VISIBLE_DEVICES=0
+ ./${workspace}/"${site_pre}${id}"/startup/start.sh ${servername} "${site_pre}${id}" &
+done
+sleep 10
+
+# start training
+echo "STARTING ANALYSIS"
+python3 ./run_fl.py --port=8003 --admin_dir="./${workspace}/${admin_username}" \
+ --run_number="${run}" --app="${algorithms_dir}/${config}" --min_clients="${n_clients}" --poc
+
+# sleep for FL system to shut down, so a new run can be started automatically
+sleep 30
diff --git a/examples/federated_analysis/virtualenv/requirements.txt b/examples/federated_analysis/virtualenv/requirements.txt
new file mode 100644
index 00000000000..863433dd015
--- /dev/null
+++ b/examples/federated_analysis/virtualenv/requirements.txt
@@ -0,0 +1,7 @@
+nvflare
+numpy
+itk-io
+monai
+pandas
+kaleido
+plotly
\ No newline at end of file
diff --git a/examples/federated_analysis/virtualenv/set_env.sh b/examples/federated_analysis/virtualenv/set_env.sh
new file mode 100755
index 00000000000..adeca22ad41
--- /dev/null
+++ b/examples/federated_analysis/virtualenv/set_env.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+export projectname='fedanalysis'
+export projectpath="."
+
+python3 -m venv ${projectname}
+source ${projectname}/bin/activate
diff --git a/examples/federated_analysis/workspaces/.gitkeep b/examples/federated_analysis/workspaces/.gitkeep
new file mode 100644
index 00000000000..8b137891791
--- /dev/null
+++ b/examples/federated_analysis/workspaces/.gitkeep
@@ -0,0 +1 @@
+
diff --git a/nvflare/private/fed/app/server/server_train.py b/nvflare/private/fed/app/server/server_train.py
index 9601b99ee42..1256985b7b8 100644
--- a/nvflare/private/fed/app/server/server_train.py
+++ b/nvflare/private/fed/app/server/server_train.py
@@ -66,7 +66,7 @@ def main():
# trainer = WorkFlowFactory().create_server_trainer(train_configs, envs)
startup = os.path.join(args.workspace, "startup")
conf = FLServerStarterConfiger(
- app_root="startup",
+ app_root=startup,
# wf_config_file_name="config_train.json",
server_config_file_name=args.fed_server,
# env_config_file_name="environment.json",
diff --git a/test/app_testing/admin_controller.py b/test/app_testing/admin_controller.py
index 66cd3d5d94a..9a52de20a5a 100644
--- a/test/app_testing/admin_controller.py
+++ b/test/app_testing/admin_controller.py
@@ -48,7 +48,6 @@ def initialize(self):
success = False
try:
- # TODO:: login or login_with_password should return FLAdminAPIResponse to be consistent?
response = None
timeout = 100
start_time = time.time()
diff --git a/test/app_testing/app_result_validator.py b/test/app_testing/app_result_validator.py
index 8f7fe9a7d5d..3e24730ae66 100644
--- a/test/app_testing/app_result_validator.py
+++ b/test/app_testing/app_result_validator.py
@@ -13,12 +13,13 @@
# limitations under the License.
import logging
+from abc import ABC, abstractmethod
-class AppResultValidator(object):
+class AppResultValidator(ABC):
def __init__(self):
- super(AppResultValidator, self).__init__()
self.logger = logging.getLogger("AppValidator")
+ @abstractmethod
def validate_results(self, server_data, client_data, run_data) -> bool:
pass
diff --git a/test/app_testing/run_app_tests.sh b/test/app_testing/run_app_tests.sh
index 7ca24059714..29b4c0ecc07 100755
--- a/test/app_testing/run_app_tests.sh
+++ b/test/app_testing/run_app_tests.sh
@@ -1,4 +1,5 @@
+#!/usr/bin/env bash
pip install tensorflow
-python test_runner.py --poc ../../nvflare/poc --n_clients 2 --yaml test_apps/test_simple.yml --app_path test_apps --cleanup
\ No newline at end of file
+python test_runner.py --poc ../../nvflare/poc --n_clients 2 --yaml test_simple.yml --app_path test_apps --cleanup
diff --git a/test/app_testing/run_example_tests.sh b/test/app_testing/run_example_tests.sh
index 6f2fc070f50..d8a4ccc0d76 100755
--- a/test/app_testing/run_example_tests.sh
+++ b/test/app_testing/run_example_tests.sh
@@ -1,4 +1,5 @@
+#!/usr/bin/env bash
pip install tensorflow
-python test_runner.py --poc ../../nvflare/poc --n_clients 2 --yaml test_apps/test_examples.yml --app_path ../../examples --cleanup
\ No newline at end of file
+python test_runner.py --poc ../../nvflare/poc --n_clients 2 --yaml test_examples.yml --app_path ../../examples --cleanup
diff --git a/test/app_testing/site_launcher.py b/test/app_testing/site_launcher.py
index 2fe9502b3d8..3406408b285 100644
--- a/test/app_testing/site_launcher.py
+++ b/test/app_testing/site_launcher.py
@@ -44,7 +44,7 @@ def __init__(
"""
This class sets up the test environment for a test. It will launch and keep track of servers and clients.
"""
- super(SiteLauncher, self).__init__()
+ super().__init__()
self.original_poc_directory = poc_directory
self.server_dir_name = server_dir_name
@@ -57,7 +57,7 @@ def __init__(
self.admin_api = None
- self.logger = logging.getLogger("SiteRunner")
+ self.logger = logging.getLogger("SiteLauncher")
# Create temporary poc directory
if not os.path.exists(self.original_poc_directory):
@@ -74,7 +74,6 @@ def __init__(
def start_server(self):
server_dir = os.path.join(self.poc_directory, self.server_dir_name)
log_path = os.path.join(server_dir, "log.txt")
- # log_file = open(log_path, 'w')
new_env = os.environ.copy()
# Create upload directory
@@ -92,7 +91,6 @@ def start_server(self):
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
- # process = subprocess.Popen(shlex.split(command, " "), preexec_fn=os.setsid, env=new_env)
print("Starting server ...")
t = threading.Thread(target=process_logs, args=(log_path, process))
@@ -100,7 +98,6 @@ def start_server(self):
self.server_properties["path"] = server_dir
self.server_properties["process"] = process
- # self.server_properties["log_file"] = log_file
self.server_properties["log_path"] = log_path
def start_clients(self, n=2):
@@ -141,7 +138,6 @@ def start_clients(self, n=2):
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
- # process = subprocess.Popen(shlex.split(command, " "), preexec_fn=os.setsid, env=new_env)
self.client_properties[client_id]["process"] = process
print(f"Launched client {client_id} process.")
@@ -223,9 +219,6 @@ def stop_all_clients(self):
def stop_all_sites(self):
self.stop_server()
- def client_status(self, client_id):
- pass
-
def finalize(self):
print(f"Deleting temporary directory: {self.poc_directory}.")
shutil.rmtree(self.poc_directory)
diff --git a/test/app_testing/test_apps/test_examples.yml b/test/app_testing/test_apps/test_examples.yml
deleted file mode 100644
index 700a34bf62a..00000000000
--- a/test/app_testing/test_apps/test_examples.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-
-
-
-tests:
- - app_name: hello-pt
- validators:
- - test.app_testing.test_apps.validators.pt_model_validator.PTModelValidator
- - app_name: hello-numpy-cross-val
- validators:
- - test.app_testing.test_apps.validators.cross_val_result_validator.CrossResultValidator
- - test.app_testing.test_apps.validators.sag_result_validator.SAGResultValidator
- - app_name: hello-numpy-sag
- validators:
- - test.app_testing.test_apps.validators.sag_result_validator.SAGResultValidator
- - app_name: hello-tf2
- validators:
- - test.app_testing.test_apps.validators.tf_model_validator.TFModelValidator
- - app_name: hello-cyclic
- validators:
- - test.app_testing.test_apps.validators.tf_model_validator.TFModelValidator
diff --git a/test/app_testing/test_apps/test_simple.yml b/test/app_testing/test_apps/test_simple.yml
deleted file mode 100644
index ae313009bb7..00000000000
--- a/test/app_testing/test_apps/test_simple.yml
+++ /dev/null
@@ -1,37 +0,0 @@
-
-
-
-tests:
- - app_name: global_model_eval
- validators:
- - test.app_testing.test_apps.validators.global_model_eval_validator.GlobalModelEvalValidator
- - test.app_testing.test_apps.validators.sag_result_validator.SAGResultValidator
- - app_name: pt
- validators:
- - test.app_testing.test_apps.validators.pt_model_validator.PTModelValidator
- - app_name: sag_weights_diff
- validators:
- - test.app_testing.test_apps.validators.sag_result_validator.SAGResultValidator
- - app_name: cross_val_one_client
- validators:
- - test.app_testing.test_apps.validators.cross_val_single_client_validator.CrossValSingleClientValidator
- - test.app_testing.test_apps.validators.sag_result_validator.SAGResultValidator
- - app_name: cross_val
- validators:
- - test.app_testing.test_apps.validators.cross_val_result_validator.CrossResultValidator
- - test.app_testing.test_apps.validators.sag_result_validator.SAGResultValidator
- - app_name: sag
- validators:
- - test.app_testing.test_apps.validators.sag_result_validator.SAGResultValidator
- - app_name: filters
- validators:
- - test.app_testing.test_apps.validators.filters_result_validator.FiltersResultValidator
- - app_name: tb_streaming
- validators:
- - test.app_testing.test_apps.validators.tb_result_validator.TBResultValidator
- - app_name: tf
- validators:
- - test.app_testing.test_apps.validators.tf_model_validator.TFModelValidator
- - app_name: cyclic
- validators:
- - test.app_testing.test_apps.validators.tf_model_validator.TFModelValidator
diff --git a/test/app_testing/test_examples.yml b/test/app_testing/test_examples.yml
new file mode 100644
index 00000000000..65ca1f0f487
--- /dev/null
+++ b/test/app_testing/test_examples.yml
@@ -0,0 +1,20 @@
+
+
+
+tests:
+ - app_name: hello-pt
+ validators:
+ - test.app_testing.validators.pt_model_validator.PTModelValidator
+ - app_name: hello-numpy-cross-val
+ validators:
+ - test.app_testing.validators.cross_val_result_validator.CrossResultValidator
+ - test.app_testing.validators.sag_result_validator.SAGResultValidator
+ - app_name: hello-numpy-sag
+ validators:
+ - test.app_testing.validators.sag_result_validator.SAGResultValidator
+ - app_name: hello-tf2
+ validators:
+ - test.app_testing.validators.tf_model_validator.TFModelValidator
+ - app_name: hello-cyclic
+ validators:
+ - test.app_testing.validators.tf_model_validator.TFModelValidator
diff --git a/test/app_testing/test_simple.yml b/test/app_testing/test_simple.yml
new file mode 100644
index 00000000000..8c3b5075137
--- /dev/null
+++ b/test/app_testing/test_simple.yml
@@ -0,0 +1,37 @@
+
+
+
+tests:
+ - app_name: global_model_eval
+ validators:
+ - test.app_testing.validators.global_model_eval_validator.GlobalModelEvalValidator
+ - test.app_testing.validators.sag_result_validator.SAGResultValidator
+ - app_name: pt
+ validators:
+ - test.app_testing.validators.pt_model_validator.PTModelValidator
+ - app_name: sag_weights_diff
+ validators:
+ - test.app_testing.validators.sag_result_validator.SAGResultValidator
+ - app_name: cross_val_one_client
+ validators:
+ - test.app_testing.validators.cross_val_single_client_validator.CrossValSingleClientValidator
+ - test.app_testing.validators.sag_result_validator.SAGResultValidator
+ - app_name: cross_val
+ validators:
+ - test.app_testing.validators.cross_val_result_validator.CrossResultValidator
+ - test.app_testing.validators.sag_result_validator.SAGResultValidator
+ - app_name: sag
+ validators:
+ - test.app_testing.validators.sag_result_validator.SAGResultValidator
+ - app_name: filters
+ validators:
+ - test.app_testing.validators.filters_result_validator.FiltersResultValidator
+ - app_name: tb_streaming
+ validators:
+ - test.app_testing.validators.tb_result_validator.TBResultValidator
+ - app_name: tf
+ validators:
+ - test.app_testing.validators.tf_model_validator.TFModelValidator
+ - app_name: cyclic
+ validators:
+ - test.app_testing.validators.tf_model_validator.TFModelValidator
diff --git a/test/app_testing/test_apps/validators/cross_val_result_validator.py b/test/app_testing/validators/cross_val_result_validator.py
similarity index 100%
rename from test/app_testing/test_apps/validators/cross_val_result_validator.py
rename to test/app_testing/validators/cross_val_result_validator.py
diff --git a/test/app_testing/test_apps/validators/cross_val_single_client_validator.py b/test/app_testing/validators/cross_val_single_client_validator.py
similarity index 97%
rename from test/app_testing/test_apps/validators/cross_val_single_client_validator.py
rename to test/app_testing/validators/cross_val_single_client_validator.py
index d41899c19c1..642f98f6e92 100644
--- a/test/app_testing/test_apps/validators/cross_val_single_client_validator.py
+++ b/test/app_testing/validators/cross_val_single_client_validator.py
@@ -13,11 +13,10 @@
# limitations under the License.
import os
-
from test.app_testing.app_result_validator import AppResultValidator
-def check_cross_validation_result(server_data, client_data, run_data):
+def check_cross_validation_result(server_data, run_data):
run_number = run_data["run_number"]
server_dir = server_data["server_path"]
@@ -79,7 +78,7 @@ def __init__(self):
def validate_results(self, server_data, client_data, run_data) -> bool:
- cross_val_result = check_cross_validation_result(server_data, client_data, run_data)
+ cross_val_result = check_cross_validation_result(server_data, run_data)
print(f"CrossVal Result: {cross_val_result}")
diff --git a/test/app_testing/test_apps/validators/filters_result_validator.py b/test/app_testing/validators/filters_result_validator.py
similarity index 100%
rename from test/app_testing/test_apps/validators/filters_result_validator.py
rename to test/app_testing/validators/filters_result_validator.py
diff --git a/test/app_testing/test_apps/validators/global_model_eval_validator.py b/test/app_testing/validators/global_model_eval_validator.py
similarity index 100%
rename from test/app_testing/test_apps/validators/global_model_eval_validator.py
rename to test/app_testing/validators/global_model_eval_validator.py
diff --git a/test/app_testing/test_apps/validators/pt_model_validator.py b/test/app_testing/validators/pt_model_validator.py
similarity index 99%
rename from test/app_testing/test_apps/validators/pt_model_validator.py
rename to test/app_testing/validators/pt_model_validator.py
index 8b8843c169d..95de8f83c4a 100644
--- a/test/app_testing/test_apps/validators/pt_model_validator.py
+++ b/test/app_testing/validators/pt_model_validator.py
@@ -14,11 +14,10 @@
import os
from collections import OrderedDict
+from test.app_testing.app_result_validator import AppResultValidator
import numpy as np
-from test.app_testing.app_result_validator import AppResultValidator
-
def check_pt_results(server_data, client_data, run_data):
run_number = run_data["run_number"]
diff --git a/test/app_testing/test_apps/validators/sag_result_validator.py b/test/app_testing/validators/sag_result_validator.py
similarity index 100%
rename from test/app_testing/test_apps/validators/sag_result_validator.py
rename to test/app_testing/validators/sag_result_validator.py
diff --git a/test/app_testing/test_apps/validators/tb_result_validator.py b/test/app_testing/validators/tb_result_validator.py
similarity index 100%
rename from test/app_testing/test_apps/validators/tb_result_validator.py
rename to test/app_testing/validators/tb_result_validator.py
diff --git a/test/app_testing/test_apps/validators/tf_model_validator.py b/test/app_testing/validators/tf_model_validator.py
similarity index 99%
rename from test/app_testing/test_apps/validators/tf_model_validator.py
rename to test/app_testing/validators/tf_model_validator.py
index 49a00b0c36a..ad0c4cd709d 100644
--- a/test/app_testing/test_apps/validators/tf_model_validator.py
+++ b/test/app_testing/validators/tf_model_validator.py
@@ -13,9 +13,8 @@
# limitations under the License.
import os
-from test.app_testing.app_result_validator import AppResultValidator
-
import pickle
+from test.app_testing.app_result_validator import AppResultValidator
def check_tf_results(server_data, client_data, run_data):
diff --git a/test/test_jsonscaner.py b/test/test_jsonscaner.py
deleted file mode 100644
index 41968a38ddf..00000000000
--- a/test/test_jsonscaner.py
+++ /dev/null
@@ -1,252 +0,0 @@
-# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import re
-import unittest
-
-from nvflare.fuel.utils.json_scanner import JsonObjectProcessor, JsonScanner, Node
-
-
-class _TestJsonProcessor(JsonObjectProcessor):
- def __init__(self):
- JsonObjectProcessor.__init__(self)
-
- def process_element(self, node: Node):
- pats = [
- r".\.pre_transforms\.#[0-9]+$",
- r"^train\.model\.name$",
- r".\.search\.#[0-9]+$",
- r".\.pre_transforms\.#[0-9]+\.args$",
- ]
- path = node.path()
- print("ENTER Level: {}; Key: {}; Pos: {}; Path: {}".format(node.level, node.key, node.position, path))
- for p in pats:
- x = re.search(p, path)
- if x:
- print("\t {} matches {}".format(path, p))
-
- node.exit_cb = self.post_process_element
-
- def post_process_element(self, node: Node):
- path = node.path()
- print("EXIT Level: {}; Key: {}; Pos: {}; Path: {}".format(node.level, node.key, node.position, path))
-
-
-def _test():
- import json
-
- test_json = """
- {
- "learning_rate": 1e-4,
- "lr_search" : [1e-4, 2e-3],
- "train": {
- "model": {
- "name": "SegAhnet",
- "args": {
- "num_classes": 2,
- "if_use_psp": false,
- "pretrain_weight_name": "{PRETRAIN_WEIGHTS_FILE}",
- "plane": "z",
- "final_activation": "softmax",
- "n_spatial_dim": 3
- },
- "search": [
- {
- "type": "float",
- "args": ["num_classes"],
- "targets": [1,3],
- "domain": "net"
- },
- {
- "type": "float",
- "args": ["n_spatial_dim"],
- "targets": [2,5],
- "domain": "net"
- },
- {
- "type": "enum",
- "args": ["n_spatial_dim", "num_classes"],
- "targets": [[2,3],[3,4],[5,1]],
- "domain": "net"
- },
- {
- "type": "enum",
- "args": ["n_spatial_dim"],
- "targets": [[2],[3],[6],[12]],
- "domain": "net"
- }
- ]
- },
- "pre_transforms": [
- {
- "name": "LoadNifti",
- "args": {
- "fields": [
- "image",
- "label"
- ]
- }
- },
- {
- "name": "ConvertToChannelsFirst",
- "args": {
- "fields": [
- "image",
- "label"
- ]
- }
- },
- {
- "name": "ScaleIntensityRange",
- "args": {
- "fields": "image",
- "a_min": -57,
- "a_max": 164,
- "b_min": 0.0,
- "b_max": 1.0,
- "clip": true
- }
- },
- {
- "name": "FastCropByPosNegRatio",
- "args": {
- "size": [
- 96,
- 96,
- 96
- ],
- "fields": "image",
- "label_field": "label",
- "pos": 1,
- "neg": 1,
- "batch_size": 3
- },
- "search": [
- {
- "domain": "transform",
- "type": "enum",
- "args": ["size"],
- "targets": [[[32, 32, 32]], [[64, 64, 64]], [[128, 128, 128]]]
- },
- {
- "domain": "transform",
- "type": "enum",
- "args": ["batch_size"],
- "targets": [[3], [4], [8], [10]]
- }
- ]
- },
- {
- "name": "RandomAxisFlip",
- "args": {
- "fields": [
- "image",
- "label"
- ],
- "probability": 0.0
- },
- "search": [
- {
- "domain": "transform",
- "type": "float",
- "args": ["probability#p"],
- "targets": [0.0, 1.0]
- },
- {
- "domain": "transform",
- "args": "DISABLED"
- }
- ]
- },
- {
- "name": "RandomRotate3D",
- "args": {
- "fields": [
- "image",
- "label"
- ],
- "probability": 0.0
- }
- },
- {
- "name": "ScaleIntensityOscillation",
- "args": {
- "fields": "image",
- "magnitude": 0.10
- }
- },
- {
- "name": "LoadNifti",
- "args": {
- "fields": [
- "image",
- "label"
- ]
- }
- },
- {
- "name": "LoadNifti",
- "args": {
- "fields": [
- "image",
- "label"
- ]
- }
- },
- {
- "name": "LoadNifti",
- "args": {
- "fields": [
- "image",
- "label"
- ]
- }
- },
- {
- "name": "LoadNifti",
- "args": {
- "fields": [
- "image",
- "label"
- ]
- }
- },
- {
- "name": "RandomAxisFlip",
- "args": {
- "fields": [
- "image",
- "label"
- ],
- "probability": 0.0
- },
- "search": [
- {
- "domain": "transform",
- "type": "float",
- "args": ["probability#p"],
- "targets": [0.0, 1.0]
- },
- {
- "domain": "transform",
- "args": "DISABLED"
- }
- ]
- }
- ]
- } }
- """
- train_config = json.loads(test_json)
- scanner = JsonScanner(train_config, "test")
- scanner.scan(_TestJsonProcessor())
diff --git a/test/controller/__init__.py b/test/unit_test/__init__.py
similarity index 100%
rename from test/controller/__init__.py
rename to test/unit_test/__init__.py
diff --git a/test/test_analytix.py b/test/unit_test/apis/test_analytix.py
similarity index 100%
rename from test/test_analytix.py
rename to test/unit_test/apis/test_analytix.py
diff --git a/test/test_fl_context.py b/test/unit_test/apis/test_fl_context.py
similarity index 100%
rename from test/test_fl_context.py
rename to test/unit_test/apis/test_fl_context.py
diff --git a/test/test_format_check.py b/test/unit_test/apis/test_format_check.py
similarity index 100%
rename from test/test_format_check.py
rename to test/unit_test/apis/test_format_check.py
diff --git a/test/test_aggregator.py b/test/unit_test/app_common/test_accumulate_weighted_aggregator.py
similarity index 100%
rename from test/test_aggregator.py
rename to test/unit_test/app_common/test_accumulate_weighted_aggregator.py
diff --git a/test/test_exclude_vars.py b/test/unit_test/app_common/test_exclude_vars.py
similarity index 100%
rename from test/test_exclude_vars.py
rename to test/unit_test/app_common/test_exclude_vars.py
diff --git a/test/test_intime_aggregator.py b/test/unit_test/app_common/test_in_time_accumulate_weighted_aggregator.py
similarity index 100%
rename from test/test_intime_aggregator.py
rename to test/unit_test/app_common/test_in_time_accumulate_weighted_aggregator.py
diff --git a/test/test_model_selection.py b/test/unit_test/app_common/test_in_time_model_selection_handler.py
similarity index 94%
rename from test/test_model_selection.py
rename to test/unit_test/app_common/test_in_time_model_selection_handler.py
index 0580b2a2b5f..030d98cde25 100644
--- a/test/test_model_selection.py
+++ b/test/unit_test/app_common/test_in_time_model_selection_handler.py
@@ -42,29 +42,29 @@ def fire_event(self, event_type: str, fl_ctx: FLContext):
return True
-class TestModelSelectionHandler:
+class TestInTimeModelSelectionHandler:
@pytest.mark.parametrize("mshandler", [IntimeModelSelectionHandler])
@pytest.mark.parametrize(
"initial,received,expected",
[
- [
+ (
1,
{
"client1": {"weight": 0.5, "iter_number": 1, "metric": 10},
},
True,
- ],
- [
+ ),
+ (
1,
{
"client1": {"weight": 0.5, "iter_number": 1, "metric": 1},
"client2": {"weight": 0.5, "iter_number": 1, "metric": 0.2},
},
False,
- ],
+ ),
],
)
- def test_model_section(self, mshandler, initial, received, expected):
+ def test_model_selection(self, mshandler, initial, received, expected):
aggregation_weights = {k: v["weight"] for k, v in received.items()}
handler = mshandler(aggregation_weights=aggregation_weights)
handler.best_val_metric = initial
diff --git a/test/test_streaming.py b/test/unit_test/app_common/test_streaming.py
similarity index 87%
rename from test/test_streaming.py
rename to test/unit_test/app_common/test_streaming.py
index 76b9d6c0fdb..14f66222886 100644
--- a/test/test_streaming.py
+++ b/test/unit_test/app_common/test_streaming.py
@@ -18,7 +18,7 @@
from nvflare.apis.dxo import DXO, DataKind
from nvflare.apis.fl_component import FLComponent
from nvflare.apis.fl_context import FLContext
-from nvflare.app_common.widgets.streaming import send_analytic_dxo, create_analytic_dxo
+from nvflare.app_common.widgets.streaming import create_analytic_dxo, send_analytic_dxo
INVALID_TEST_CASES = [
(list(), dict(), FLContext(), TypeError, f"expect comp to be an instance of FLComponent, but got {type(list())}"),
@@ -34,8 +34,14 @@
INVALID_WRITE_TEST_CASES = [
(list(), 1.0, AnalyticsDataType.SCALAR, TypeError, f"expect tag to be an instance of str, but got {type(list())}"),
- ("tag", list(), AnalyticsDataType.SCALAR, TypeError, f"expect value to be an instance of float, but got {type(list())}"),
- (list(), 1.0, AnalyticsDataType.SCALARS, TypeError, f"expect tag to be an instance of str, but got {type(list())}"),
+ (
+ "tag",
+ list(),
+ AnalyticsDataType.SCALAR,
+ TypeError,
+ f"expect value to be an instance of float, but got {type(list())}",
+ ),
+ (list(), 1.0, AnalyticsDataType.SCALARS, TypeError, f"expect tag to be an instance of str, but got {type(list())}"),
("tag", 1.0, AnalyticsDataType.SCALARS, TypeError, f"expect value to be an instance of dict, but got {type(1.0)}"),
(list(), 1.0, AnalyticsDataType.TEXT, TypeError, f"expect tag to be an instance of str, but got {type(list())}"),
("tag", 1.0, AnalyticsDataType.TEXT, TypeError, f"expect value to be an instance of str, but got {type(1.0)}"),
diff --git a/test/unit_test/controller/__init__.py b/test/unit_test/controller/__init__.py
new file mode 100644
index 00000000000..2b8f6c7e874
--- /dev/null
+++ b/test/unit_test/controller/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/test/controller/controller_test.py b/test/unit_test/controller/controller_test.py
similarity index 100%
rename from test/controller/controller_test.py
rename to test/unit_test/controller/controller_test.py
diff --git a/test/controller/test_basic.py b/test/unit_test/controller/test_basic.py
similarity index 100%
rename from test/controller/test_basic.py
rename to test/unit_test/controller/test_basic.py
diff --git a/test/controller/test_broadcast_behavior.py b/test/unit_test/controller/test_broadcast_behavior.py
similarity index 100%
rename from test/controller/test_broadcast_behavior.py
rename to test/unit_test/controller/test_broadcast_behavior.py
diff --git a/test/controller/test_callback.py b/test/unit_test/controller/test_callback.py
similarity index 100%
rename from test/controller/test_callback.py
rename to test/unit_test/controller/test_callback.py
diff --git a/test/controller/test_invalid_input.py b/test/unit_test/controller/test_invalid_input.py
similarity index 99%
rename from test/controller/test_invalid_input.py
rename to test/unit_test/controller/test_invalid_input.py
index 1383cfba0cf..4e1fa61cd78 100644
--- a/test/controller/test_invalid_input.py
+++ b/test/unit_test/controller/test_invalid_input.py
@@ -12,9 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import pytest
import re
+import pytest
+
from nvflare.apis.controller_spec import SendOrder
from nvflare.apis.fl_context import FLContext
from nvflare.apis.shareable import Shareable
diff --git a/test/controller/test_relay_behavior.py b/test/unit_test/controller/test_relay_behavior.py
similarity index 100%
rename from test/controller/test_relay_behavior.py
rename to test/unit_test/controller/test_relay_behavior.py
diff --git a/test/controller/test_send_bahavior.py b/test/unit_test/controller/test_send_bahavior.py
similarity index 100%
rename from test/controller/test_send_bahavior.py
rename to test/unit_test/controller/test_send_bahavior.py
diff --git a/test/controller/test_task.py b/test/unit_test/controller/test_task.py
similarity index 100%
rename from test/controller/test_task.py
rename to test/unit_test/controller/test_task.py
diff --git a/test/controller/test_task_mgmt.py b/test/unit_test/controller/test_task_mgmt.py
similarity index 100%
rename from test/controller/test_task_mgmt.py
rename to test/unit_test/controller/test_task_mgmt.py
diff --git a/test/unit_test/fuel/test_json_scanner.py b/test/unit_test/fuel/test_json_scanner.py
new file mode 100644
index 00000000000..878e61d2698
--- /dev/null
+++ b/test/unit_test/fuel/test_json_scanner.py
@@ -0,0 +1,250 @@
+# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import re
+
+from nvflare.fuel.utils.json_scanner import JsonObjectProcessor, JsonScanner, Node
+
+test_json = """
+{
+ "learning_rate": 1e-4,
+ "lr_search" : [1e-4, 2e-3],
+ "train": {
+ "model": {
+ "name": "SegAhnet",
+ "args": {
+ "num_classes": 2,
+ "if_use_psp": false,
+ "pretrain_weight_name": "{PRETRAIN_WEIGHTS_FILE}",
+ "plane": "z",
+ "final_activation": "softmax",
+ "n_spatial_dim": 3
+ },
+ "search": [
+ {
+ "type": "float",
+ "args": ["num_classes"],
+ "targets": [1,3],
+ "domain": "net"
+ },
+ {
+ "type": "float",
+ "args": ["n_spatial_dim"],
+ "targets": [2,5],
+ "domain": "net"
+ },
+ {
+ "type": "enum",
+ "args": ["n_spatial_dim", "num_classes"],
+ "targets": [[2,3],[3,4],[5,1]],
+ "domain": "net"
+ },
+ {
+ "type": "enum",
+ "args": ["n_spatial_dim"],
+ "targets": [[2],[3],[6],[12]],
+ "domain": "net"
+ }
+ ]
+ },
+ "pre_transforms": [
+ {
+ "name": "LoadNifti",
+ "args": {
+ "fields": [
+ "image",
+ "label"
+ ]
+ }
+ },
+ {
+ "name": "ConvertToChannelsFirst",
+ "args": {
+ "fields": [
+ "image",
+ "label"
+ ]
+ }
+ },
+ {
+ "name": "ScaleIntensityRange",
+ "args": {
+ "fields": "image",
+ "a_min": -57,
+ "a_max": 164,
+ "b_min": 0.0,
+ "b_max": 1.0,
+ "clip": true
+ }
+ },
+ {
+ "name": "FastCropByPosNegRatio",
+ "args": {
+ "size": [
+ 96,
+ 96,
+ 96
+ ],
+ "fields": "image",
+ "label_field": "label",
+ "pos": 1,
+ "neg": 1,
+ "batch_size": 3
+ },
+ "search": [
+ {
+ "domain": "transform",
+ "type": "enum",
+ "args": ["size"],
+ "targets": [[[32, 32, 32]], [[64, 64, 64]], [[128, 128, 128]]]
+ },
+ {
+ "domain": "transform",
+ "type": "enum",
+ "args": ["batch_size"],
+ "targets": [[3], [4], [8], [10]]
+ }
+ ]
+ },
+ {
+ "name": "RandomAxisFlip",
+ "args": {
+ "fields": [
+ "image",
+ "label"
+ ],
+ "probability": 0.0
+ },
+ "search": [
+ {
+ "domain": "transform",
+ "type": "float",
+ "args": ["probability#p"],
+ "targets": [0.0, 1.0]
+ },
+ {
+ "domain": "transform",
+ "args": "DISABLED"
+ }
+ ]
+ },
+ {
+ "name": "RandomRotate3D",
+ "args": {
+ "fields": [
+ "image",
+ "label"
+ ],
+ "probability": 0.0
+ }
+ },
+ {
+ "name": "ScaleIntensityOscillation",
+ "args": {
+ "fields": "image",
+ "magnitude": 0.10
+ }
+ },
+ {
+ "name": "LoadNifti",
+ "args": {
+ "fields": [
+ "image",
+ "label"
+ ]
+ }
+ },
+ {
+ "name": "LoadNifti",
+ "args": {
+ "fields": [
+ "image",
+ "label"
+ ]
+ }
+ },
+ {
+ "name": "LoadNifti",
+ "args": {
+ "fields": [
+ "image",
+ "label"
+ ]
+ }
+ },
+ {
+ "name": "LoadNifti",
+ "args": {
+ "fields": [
+ "image",
+ "label"
+ ]
+ }
+ },
+ {
+ "name": "RandomAxisFlip",
+ "args": {
+ "fields": [
+ "image",
+ "label"
+ ],
+ "probability": 0.0
+ },
+ "search": [
+ {
+ "domain": "transform",
+ "type": "float",
+ "args": ["probability#p"],
+ "targets": [0.0, 1.0]
+ },
+ {
+ "domain": "transform",
+ "args": "DISABLED"
+ }
+ ]
+ }
+]
+} }
+"""
+TRAIN_CONFIG = json.loads(test_json)
+
+
+def _post_process_element(node: Node):
+ path = node.path()
+ print("EXIT Level: {}; Key: {}; Pos: {}; Path: {}".format(node.level, node.key, node.position, path))
+
+
+class _TestJsonProcessor(JsonObjectProcessor):
+ def process_element(self, node: Node):
+ pats = [
+ r".\.pre_transforms\.#[0-9]+$",
+ r"^train\.model\.name$",
+ r".\.search\.#[0-9]+$",
+ r".\.pre_transforms\.#[0-9]+\.args$",
+ ]
+ path = node.path()
+ print("ENTER Level: {}; Key: {}; Pos: {}; Path: {}".format(node.level, node.key, node.position, path))
+ for p in pats:
+ x = re.search(p, path)
+ if x:
+ print("\t {} matches {}".format(path, p))
+
+ node.exit_cb = _post_process_element
+
+
+class TestJsonScanner:
+ def test_scan(self):
+ scanner = JsonScanner(TRAIN_CONFIG, "test")
+ scanner.scan(_TestJsonProcessor())
diff --git a/test/test_security_utils.py b/test/unit_test/fuel/test_security_utils.py
similarity index 100%
rename from test/test_security_utils.py
rename to test/unit_test/fuel/test_security_utils.py
diff --git a/test/test_messageproto.py b/test/unit_test/private/test_messageproto.py
similarity index 100%
rename from test/test_messageproto.py
rename to test/unit_test/private/test_messageproto.py
diff --git a/test/utils.py b/test/unit_test/utils.py
similarity index 100%
rename from test/utils.py
rename to test/unit_test/utils.py