diff --git a/3rdParty/itk-io.LICENSE.txt b/3rdParty/itk-io.LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/3rdParty/itk-io.LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/3rdParty/kaleido.LICENSE.txt b/3rdParty/kaleido.LICENSE.txt new file mode 100644 index 00000000000..904dd47bba6 --- /dev/null +++ b/3rdParty/kaleido.LICENSE.txt @@ -0,0 +1,267 @@ +All code developed in this repository is released under the MIT license (1). + +The Kaleido executable includes Chromium, which is released under the 3-clause BSD license (2). + +Chromium includes a wide range of third-party dependencies with varying licenses. +See the CREDITS.html file distributed with this package for license details +of these dependencies. + +The CREDITS.html is also available at https://github.com/plotly/Kaleido/blob/master/repos/CREDITS.html + +The Kaleido python PyPI package vendors MathJax, which is released under the terms of the Apache License (3) + +(1) ---- +The MIT License (MIT) + +Copyright (c) 2020 Plotly, Inc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +(2) ---- +// Copyright 2015 The Chromium Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +(3) ---- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/3rdParty/monai.LICENSE.txt b/3rdParty/monai.LICENSE.txt new file mode 100644 index 00000000000..261eeb9e9f8 --- /dev/null +++ b/3rdParty/monai.LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/3rdParty/pandas.LICENSE.txt b/3rdParty/pandas.LICENSE.txt new file mode 100644 index 00000000000..a0cc369f725 --- /dev/null +++ b/3rdParty/pandas.LICENSE.txt @@ -0,0 +1,31 @@ +BSD 3-Clause License + +Copyright (c) 2008-2011, AQR Capital Management, LLC, Lambda Foundry, Inc. and PyData Development Team +All rights reserved. + +Copyright (c) 2011-2021, Open source contributors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/3rdParty/plotly.LICENSE.txt b/3rdParty/plotly.LICENSE.txt new file mode 100644 index 00000000000..98b37b6f421 --- /dev/null +++ b/3rdParty/plotly.LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016-2018 Plotly, Inc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/docs/examples/hello_pt.rst b/docs/examples/hello_pt.rst index 8899eb03387..93ba9015404 100644 --- a/docs/examples/hello_pt.rst +++ b/docs/examples/hello_pt.rst @@ -4,7 +4,7 @@ Quickstart (PyTorch) Before You Start ---------------- -Feel free to refer to the official :doc:`documentation <../programming_guide>` at any point to learn more about the specifics of `NVIDIA FLARE `_. +Feel free to refer to the :doc:`detailed documentation <../programming_guide>` at any point to learn more about the specifics of `NVIDIA FLARE `_. Make sure you have an environment with NVIDIA FLARE installed. You can follow the :doc:`installation <../installation>` guide on the general concept of Python virtual environment (the recommended environment) and how to diff --git a/docs/faq.rst b/docs/faq.rst index 382f1a75ede..7248c791f4e 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -387,3 +387,5 @@ Known issues large to be uploaded and that causes timeout. #. Please don't start a new training run or start a new app before the previous application is fully stopped. Users can do ``abort client`` and ``abort server`` before ``start_app`` for the new run. +#. After calling ``shutdown client`` for a client running multi GPUs, a process (sub_worker_process) may remain. The + work around for this is to run ``abort client`` before the ``shutdown`` command. diff --git a/docs/programming_guide/controllers.rst b/docs/programming_guide/controllers.rst index 3f8d5272253..5b26c4785e2 100644 --- a/docs/programming_guide/controllers.rst +++ b/docs/programming_guide/controllers.rst @@ -9,15 +9,12 @@ Controller/Worker Interactions NVIDIA FLARE 2.0's collaborative computing is achieved through the Controller/Worker interactions. The following diagram shows how the Controller and Worker interact. -.. image:: ../resources/Controller_worker.png +.. image:: ../resources/Controller.png :height: 300px The Controller is a python object that controls or coordinates the Workers to get a job done. The controller is run on the FL server (highlighted on the right). -.. image:: ../resources/Controller.png - :height: 300px - A Worker is capable of performing tasks. Workers run on FL clients. In its control logic, the Controller assigns tasks to Workers and processes task results from the Workers. diff --git a/docs/resources/Controller_worker.png b/docs/resources/Controller_worker.png deleted file mode 100644 index 39f7b695e08..00000000000 Binary files a/docs/resources/Controller_worker.png and /dev/null differ diff --git a/examples/README.md b/examples/README.md index adf4dea7439..554a3e6c4cd 100644 --- a/examples/README.md +++ b/examples/README.md @@ -17,6 +17,8 @@ The provided examples cover different aspects of [NVIDIA FLARE](https://nvidia.g ### 1.2 Deep Learning * [Hello PyTorch](./hello-pt/README.md) * Example using [NVIDIA FLARE](https://nvidia.github.io/NVFlare) an image classifier using [FedAvg]([FedAvg](https://arxiv.org/abs/1602.05629)) and [PyTorch](https://pytorch.org/) as the deep learning training framework. +* [Hello PyTorch with TensorBoard](./hello-pt-tb/README.md) + * Example building upon [Hello PyTorch](./hello-pt/README.md) showcasing the [TensorBoard](https://tensorflow.org/tensorboard) streaming capability from the clients to the server. * [Hello TensorFlow](./hello-tf2/README.md) * Example of using [NVIDIA FLARE](https://nvidia.github.io/NVFlare) an image classifier using [FedAvg]([FedAvg](https://arxiv.org/abs/1602.05629)) and [TensorFlow](https://tensorflow.org/) as the deep learning training framework. @@ -30,4 +32,6 @@ The provided examples cover different aspects of [NVIDIA FLARE](https://nvidia.g * [Federated Learning with Differential Privacy for BraTS18 segmentation](./brats18/README.md) * Illustrates the use of differential privacy for training brain tumor segmentation models using federated learning. * [Federated Learning for Prostate Segmentation from Multi-source Data](./prostate/README.md) - * Example of training a multi-institutional prostate segmentation model using [FedAvg](https://arxiv.org/abs/1602.05629), [FedProx](https://arxiv.org/abs/1812.06127), and [Ditto](https://arxiv.org/abs/2012.04221). + * Example of training a multi-institutional prostate segmentation model using [FedAvg](https://arxiv.org/abs/1602.05629), [FedProx](https://arxiv.org/abs/1812.06127), and [Ditto](https://arxiv.org/abs/2012.04221). +* [Federated Analysis](./federated_analysis/README.md) + * Example of gathering local data summary statistics to compute the global dataset statistics. diff --git a/examples/cifar10/figs/plot_tensorboard_events.py b/examples/cifar10/figs/plot_tensorboard_events.py index ee5b331e763..1d35d0fffe9 100644 --- a/examples/cifar10/figs/plot_tensorboard_events.py +++ b/examples/cifar10/figs/plot_tensorboard_events.py @@ -76,7 +76,7 @@ def main(): data = {"Config": [], "Step": [], "Accuracy": []} if add_cross_site_val: - xsite_keys = ["SRV_server", "SRV_server_best"] + xsite_keys = ["SRV_FL_global_model.pt", "SRV_best_FL_global_model.pt"] xsite_data = {"Config": []} for k in xsite_keys: xsite_data.update({k: []}) @@ -94,7 +94,7 @@ def main(): if add_cross_site_val: xsite_file = glob.glob( - os.path.join(server_results_root, exp["run"] + "/**/cross_site_val.json"), recursive=True + os.path.join(server_results_root, exp["run"] + "/**/cross_val_results.json"), recursive=True ) assert len(xsite_file) == 1, "No unique x-site file found!" with open(xsite_file[0], "r") as f: diff --git a/examples/federated_analysis/.gitignore b/examples/federated_analysis/.gitignore new file mode 100644 index 00000000000..e8e7d950386 --- /dev/null +++ b/examples/federated_analysis/.gitignore @@ -0,0 +1,23 @@ +# ide +.idea/ +.ipynb_checkpoints/ + +# nvflare artifacts +log.txt +client_token.txt +*.fl +audit.log +transfer +workspaces + +# python +__pycache__ +.pyc + +# virtual environments +fedanalysis + +# data +dataset +dataset* +*results* diff --git a/examples/federated_analysis/README.md b/examples/federated_analysis/README.md new file mode 100644 index 00000000000..6d67062d96e --- /dev/null +++ b/examples/federated_analysis/README.md @@ -0,0 +1,94 @@ +# Federated Analysis with NVIDIA FLARE + +## (Optional) 0. Set up a virtual environment +``` +python3 -m pip install --user --upgrade pip +python3 -m pip install --user virtualenv +``` +(If needed) make all shell scripts executable using +``` +find . -name ".sh" -exec chmod +x {} \; +``` +initialize virtual environment. +``` +source ./virtualenv/set_env.sh +``` +install required packages. +``` +pip install --upgrade pip +pip install -r ./virtualenv/requirements.txt +``` + +## 1. Download the example data + +As an example, you can download the dataset from the ["COVID-19 Radiography Database"](https://www.kaggle.com/tawsifurrahman/covid19-radiography-database). +Download the `archive.zip` and extract to `./data/.`. + +Next, create the data lists simulating different clients with varying amounts and types of images. +The downloaded archive contains subfolders for four different classes: `COVID`, `Lung_Opacity`, `Normal`, and `Viral Pneumonia`. +Here we assume each class of image corresponds to a different sites. +``` +python3 data/prepare_data.py --input_dir ./data +``` + +With this ratio setting, site-3 will have the largest number of images. You should see the following output +``` +Created 4 data lists for ['COVID', 'Lung_Opacity', 'Normal', 'Viral Pneumonia']. +Saved 3616 entries at ./data/site-1_COVID.json +Saved 6012 entries at ./data/site-2_Lung_Opacity.json +Saved 10192 entries at ./data/site-3_Normal.json +Saved 1345 entries at ./data/site-4_Viral Pneumonia.json +``` + +## 2. Create your POC workspace +To run FL experiments in POC mode, create your local FL workspace the below command. +In the following experiments, we will be using three clients. One for each data list prepared above. Press "y" when prompted. +``` +./create_poc_workpace.sh 4 +``` + +## 3. Compute the local and global intensity histograms + +First, we add the current directory path to `config_fed_client.json` files to generate the absolute path for `data_root`. +``` +sed -i "s|PWD|${PWD}|g" configs/fed_analysis/config/config_fed_client.json +``` +Next, we start the federated analysis by executing the [run_poc.sh](./run_poc.sh) script. This will start the federated workflow automatically, execute the tasks on the clients and gather the histograms on the server. + +**Note:** The [run_poc.sh](./run_poc.sh) script follows this pattern: `./run_poc.sh [n_clients] [config] [run]` + +FOr example, to run three clients and under `run_1`, run the following: +``` +./run_poc.sh 4 fed_analysis 1 +``` + +**Note:** This example uses the [k-anonymity](https://en.wikipedia.org/wiki/K-anonymity) approach to ensure that no individual patient's data is leaked to the server. +Clients will only send intensity histogram statistics if computed on at least `k` images. The default number is set by `min_images=10` in `AnalysisExecutor`. + +Other default parameters of the `AnalysisExecutor` are chosen to work well with the used example data. For other datasets, the histogram parameters (`n_bins`, `range_min`, and `range_max`) might need to be adjusted. + +## 4. Visualize the result + +If successful, the computed histograms will be shown in the `run_*` folders as `histograms.html` and `histograms.svg`. +For example, the gathered local and global histograms will look like this. + +![Example local and global histograms](./histograms_example.svg) + +## 5. Get results using the admin client + +In real-world FL scenarios, the researcher might not have direct access to the server machine. Hence, the researcher can use the admin client console to control the experimentation. See [here](https://nvidia.github.io/NVFlare/user_guide/admin_commands.html) for details. +After completing the federated analysis run, you can check the histogram files have been created: +``` +> ls server run_1 +app_server +fl_app.txt +histograms.html +histograms.svg +``` +The result can be downloaded to the admin's machine using this command: +``` +> download_folder ../run_1 +``` + +After download, the files will be available in the admin workspace under `transfer`. + diff --git a/examples/federated_analysis/configs/fed_analysis/config/config_fed_client.json b/examples/federated_analysis/configs/fed_analysis/config/config_fed_client.json new file mode 100644 index 00000000000..a9219e0130e --- /dev/null +++ b/examples/federated_analysis/configs/fed_analysis/config/config_fed_client.json @@ -0,0 +1,17 @@ +{ + "format_version": 2, + "executors": [ + { + "tasks": ["histogram"], + "executor": { + "path": "analysis_executor.AnalysisExecutor", + "args": { + "data_root": "PWD/data" + } + } + } + ], + "task_result_filters": [], + "task_data_filters": [], + "components": [] +} diff --git a/examples/federated_analysis/configs/fed_analysis/config/config_fed_server.json b/examples/federated_analysis/configs/fed_analysis/config/config_fed_server.json new file mode 100644 index 00000000000..a00376e46e5 --- /dev/null +++ b/examples/federated_analysis/configs/fed_analysis/config/config_fed_server.json @@ -0,0 +1,18 @@ +{ + "format_version": 2, + "server": { + "heart_beat_timeout": 600 + }, + "task_data_filters": [], + "task_result_filters": [], + "workflows": [ + { + "id": "server_workflow", + "path": "analysis_controller.AnalysisController", + "args": { + "min_clients": 4 + } + } + ], + "components": [] +} diff --git a/examples/federated_analysis/configs/fed_analysis/custom/analysis_controller.py b/examples/federated_analysis/configs/fed_analysis/custom/analysis_controller.py new file mode 100644 index 00000000000..3441443e622 --- /dev/null +++ b/examples/federated_analysis/configs/fed_analysis/custom/analysis_controller.py @@ -0,0 +1,172 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import numpy as np +import plotly.graph_objects as go +from analysis_executor import SupportedTasks +from plotly.subplots import make_subplots + +from nvflare.apis.client import Client +from nvflare.apis.dxo import from_shareable +from nvflare.apis.fl_constant import FLContextKey, ReturnCode +from nvflare.apis.fl_context import FLContext +from nvflare.apis.impl.controller import ClientTask, Controller, Task +from nvflare.apis.shareable import Shareable +from nvflare.apis.signal import Signal + + +class AnalysisController(Controller): + def __init__(self, min_clients: int = 1): + """Controller for federated analysis. + + Args: + min_clients: how many statistics to gather before computing the global statisitcs. + """ + super().__init__() + self.histograms = dict() + self._min_clients = min_clients + self.run_dir = None + + def start_controller(self, fl_ctx: FLContext): + self.run_dir = os.path.join(fl_ctx.get_prop(FLContextKey.APP_ROOT), "..") + + def stop_controller(self, fl_ctx: FLContext): + pass + + def process_result_of_unknown_task( + self, + client: Client, + task_name: str, + client_task_id: str, + result: Shareable, + fl_ctx: FLContext, + ): + self.log_warning(fl_ctx, f"Unknown task: {task_name} from client {client.name}.") + + def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): + self.log_info(fl_ctx, "Analysis control flow started.") + if abort_signal.triggered: + return + task = Task(name=SupportedTasks.HISTOGRAM, data=Shareable(), result_received_cb=self._process_result_histogram) + self.broadcast_and_wait( + task=task, + min_responses=self._min_clients, + fl_ctx=fl_ctx, + wait_time_after_min_received=1, + abort_signal=abort_signal, + ) + if abort_signal.triggered: + return + + # Visualize the histograms + self._create_tb_histograms(fl_ctx) + + self.log_info(fl_ctx, "Analysis control flow finished.") + + def _process_result_histogram(self, client_task: ClientTask, fl_ctx: FLContext): + task_name = client_task.task.name + client_name = client_task.client.name + self.log_info(fl_ctx, f"Processing {task_name} result from client {client_name}") + result = client_task.result + rc = result.get_return_code() + + if rc == ReturnCode.OK: + dxo = from_shareable(result) + data_stat_dict = dxo.data + self.log_info(fl_ctx, f"Received result entries {data_stat_dict.keys()}") + self.histograms.update({client_name: data_stat_dict}) + if "histogram" in data_stat_dict.keys(): + self.log_info(fl_ctx, f"Client {client_name} finished {task_name} returned histogram.") + else: + self.log_info(fl_ctx, f"Client {client_name} finished {task_name} but return no histogram.") + else: + self.log_error(fl_ctx, f"Ignore the client train result. {task_name} tasked returned error code: {rc}") + + # Cleanup task result + client_task.result = None + + def _create_tb_histograms(self, fl_ctx: FLContext): + global_histogram = None + global_bin_edges = None + global_n_images = 0 + global_n_included_images = 0 + + n_clients = len(self.histograms.keys()) + # compute global histogram and plot local ones. + if n_clients == 0: + self.log_warning(fl_ctx, "There are no histograms!") + return + + fig = make_subplots(rows=2, cols=n_clients) + n_plots = 0 + + i = 0 + for client_name, _histo in self.histograms.items(): + global_n_images += _histo.get("n_images", 0) + global_n_included_images += _histo.get("n_included_images", 0) + + # compute global histogram + if "histogram" in _histo: + if global_histogram is None: + global_histogram = _histo["histogram"] + global_bin_edges = _histo["bin_edges"] + else: # add to current histogram + if np.all(np.equal(_histo["bin_edges"], global_bin_edges)): + global_histogram += _histo["histogram"] + else: + self.log_warning( + fl_ctx, + f"bin edges don't match the initial global bin edges. " + f"Ignoring results from {client_name} in global histogram.", + ) + + # write local histogram to TensorBoard + i += 1 + fig.add_trace( + go.Scatter( + x=_histo["bin_edges"], + y=_histo["histogram"], + name=f"{client_name} " + f"({_histo.get('n_included_images', 0)} of {_histo.get('n_images', 0)} images)", + mode="lines", + fill="tozeroy", + ), + row=1, + col=i, + ) + n_plots += 1 + if global_histogram is not None: + fig.add_trace( + go.Scatter( + x=global_bin_edges, + y=global_histogram, + name=f"Global ({global_n_included_images} of {global_n_images} images)", + mode="lines", + fill="tozeroy", + ), + row=2, + col=1, + ) + + fig.update_layout( + height=600, + width=800, + title_text=f"Histograms for {n_plots} of {n_clients} clients", + xaxis_title="Image Intensity", + yaxis_title="Count", + ) + fig.write_html(os.path.join(self.run_dir, "histograms.html")) + fig.write_image(os.path.join(self.run_dir, "histograms.svg")) diff --git a/examples/federated_analysis/configs/fed_analysis/custom/analysis_executor.py b/examples/federated_analysis/configs/fed_analysis/custom/analysis_executor.py new file mode 100644 index 00000000000..639ecdf349f --- /dev/null +++ b/examples/federated_analysis/configs/fed_analysis/custom/analysis_executor.py @@ -0,0 +1,157 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import os + +import numpy as np +from monai.data import ITKReader, load_decathlon_datalist +from monai.transforms import LoadImage + +from nvflare.apis.dxo import DXO, DataKind +from nvflare.apis.executor import Executor +from nvflare.apis.fl_constant import ReservedKey, ReturnCode +from nvflare.apis.fl_context import FLContext +from nvflare.apis.shareable import Shareable, make_reply +from nvflare.apis.signal import Signal + + +class SupportedTasks(object): + HISTOGRAM = "histogram" + + +class AnalysisExecutor(Executor): + def __init__( + self, + data_root: str = "./data", + data_list_key: str = "data", + min_images: int = 10, + n_bins: int = 256, + range_min: float = 0.0, + range_max: float = 255.0, # assumes BMP/PNG/JPEGSs as input + ): + """Executor for federated analysis. + + Args: + data_root: directory with local image data. + data_list_key: data list key to use. + min_images: minimum number of images needed to compute histogram (if less images are included in the computation, no statistics will be sent to the server). + n_bins: number of bins for the histogram. + range_min: minimum intensity value to include in the histogram. + range_max: maximum intensity value to include in the histogram + + Returns: + a Shareable with the computed statistics after `execute()` + """ + super().__init__() + self.data_list_key = data_list_key + self.data_root = data_root + self.data = None + + self.n_bins = n_bins + self.range_min = range_min + self.range_max = range_max + + self.data_list = None + + self._min_images = min_images + + self.loader = LoadImage() + self.loader.register(ITKReader()) + + def _load_data_list(self, client_name, fl_ctx: FLContext): + dataset_json = glob.glob(os.path.join(self.data_root, client_name + "*.json")) + if len(dataset_json) != 1: + self.log_error( + fl_ctx, f"No unique matching dataset list found in {self.data_root} for client {client_name}" + ) + return False + dataset_json = dataset_json[0] + self.log_info(fl_ctx, f"Reading data from {dataset_json}") + self.data_list = load_decathlon_datalist( + data_list_file_path=dataset_json, data_list_key=self.data_list_key, base_dir=self.data_root + ) + self.log_info(fl_ctx, f"Client {client_name} has {len(self.data_list)} images") + return True + + def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort_signal: Signal) -> Shareable: + self.log_info(fl_ctx, f"Executing {task_name}") + try: + client_name = fl_ctx.get_prop(ReservedKey.CLIENT_NAME) + if not self._load_data_list(client_name, fl_ctx): + self.log_error(fl_ctx, f"Reading data list for client {client_name} failed!") + return make_reply(ReturnCode.ERROR) + + if task_name == SupportedTasks.HISTOGRAM: + result_dict = self._compute_histo(fl_ctx, abort_signal) + if abort_signal.triggered: + return make_reply(ReturnCode.TASK_ABORTED) + + if result_dict: + dxo = DXO(data_kind=DataKind.METRICS, data=result_dict) + return dxo.to_shareable() + else: + return make_reply(ReturnCode.EXECUTION_EXCEPTION) + + else: + self.log_error(fl_ctx, f"{task_name} is not a supported task.") + return make_reply(ReturnCode.TASK_UNKNOWN) + except BaseException as e: + self.log_exception(fl_ctx, f"Task {task_name} failed. Exception: {e.__str__()}") + return make_reply(ReturnCode.EXECUTION_EXCEPTION) + + def _compute_histo(self, fl_ctx: FLContext, abort_signal: Signal): + n_images = len(self.data_list) + n_included_images = 0 + bin_edges = [] + + if n_images < self._min_images: # don't send stats if too few cases available + return { + "n_images": n_images, + "n_included_images": n_included_images, + } + + histogram = np.zeros((self.n_bins,), dtype=np.int64) + for i, entry in enumerate(self.data_list): # TODO: use multi-processing + if abort_signal.triggered: + return None + file = entry.get("image") + + try: + img, meta = self.loader(file) + curr_histogram, bin_edges = np.histogram(img, bins=self.n_bins, range=(self.range_min, self.range_max)) + histogram += curr_histogram + n_included_images += 1 + + if i % 100 == 0: + self.log_info(fl_ctx, f"adding {i + 1} of {len(self.data_list)}: {file}") + except BaseException as e: + self.log_exception( + fl_ctx, f"Failed to load file {file} with exception: {e.__str__()}. Skipping this image..." + ) + + if n_included_images < self._min_images: # don't send stats if too few cases were included + return { + "n_images": n_images, + "n_included_images": n_included_images, + } + + self.log_info(fl_ctx, f"Computed histogram for {n_included_images} of {n_images} images.") + + return { + "n_images": n_images, + "n_included_images": n_included_images, + "histogram": histogram, + "bin_edges": bin_edges, + } diff --git a/examples/federated_analysis/create_poc_workpace.sh b/examples/federated_analysis/create_poc_workpace.sh new file mode 100755 index 00000000000..396b57beb58 --- /dev/null +++ b/examples/federated_analysis/create_poc_workpace.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +workspace="poc_workspace" + +n_clients=$1 + +if test -z "${n_clients}" +then + echo "Usage: ./create_poc_workspace.sh [n_clients], e.g. ./create_poc_workspace.sh 4" + exit 1 +fi + +cur_dir=${PWD} + +# create POC startup kits +cd "workspaces" || exit +python3 -m nvflare.lighter.poc -n "${n_clients}" || exit +# There should be $n_clients site-N folders. + +# move the folder +mv "poc" ${workspace} +echo "Created POC workspace at ./workspaces/${workspace}" + +cd "${cur_dir}" || exit diff --git a/examples/federated_analysis/data/prepare_data.py b/examples/federated_analysis/data/prepare_data.py new file mode 100644 index 00000000000..d5d764c95d1 --- /dev/null +++ b/examples/federated_analysis/data/prepare_data.py @@ -0,0 +1,86 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import glob +import json +import os +import random + +SEED = 0 + + +def create_datasets(root, subdirs, extension, shuffle, seed): + random.seed(seed) + + data_lists = [] + for subdir in subdirs: + search_string = os.path.join(root, "**", subdir, "*" + extension) + data_list = glob.glob(search_string, recursive=True) + + assert ( + len(data_list) > 0 + ), f"No images found using {search_string} for subdir '{subdir}' and extension '{extension}'!" + + if shuffle: + random.shuffle(data_list) + + data_lists.append(data_list) + + return data_lists + + +def save_data_list(data, data_list_file, data_root, key="data"): + data_list = [] + for d in data: + data_list.append({"image": d.replace(data_root + os.path.sep, "")}) + + os.makedirs(os.path.dirname(data_list_file), exist_ok=True) + with open(data_list_file, "w") as f: + json.dump({key: data_list}, f, indent=4) + + print(f"Saved {len(data_list)} entries at {data_list_file}") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--input_dir", type=str, required=True, help="Location of image files") + parser.add_argument("--input_ext", type=str, default=".png", help="Search extions") + parser.add_argument("--output_dir", type=str, default="./data", help="Output location of data lists") + parser.add_argument( + "--subdirs", + type=str, + default="COVID,Lung_Opacity,Normal,Viral Pneumonia", + help="A list of subfolders to include.", + ) + args = parser.parse_args() + + assert "," in args.subdirs, "Expecting a comma separated list of subdirs names" + subdirs = [sd for sd in args.subdirs.split(",")] + + data_lists = create_datasets( + root=args.input_dir, subdirs=subdirs, extension=args.input_ext, shuffle=True, seed=SEED + ) + print(f"Created {len(data_lists)} data lists for {subdirs}.") + + site_id = 1 + for subdir, data_list in zip(subdirs, data_lists): + save_data_list( + data_list, os.path.join(args.output_dir, f"site-{site_id}_{subdir}.json"), data_root=args.input_dir + ) + site_id += 1 + + +if __name__ == "__main__": + main() diff --git a/examples/federated_analysis/histograms_example.html b/examples/federated_analysis/histograms_example.html new file mode 100644 index 00000000000..7e91052f0f1 --- /dev/null +++ b/examples/federated_analysis/histograms_example.html @@ -0,0 +1,71 @@ + + + +
+
+ + \ No newline at end of file diff --git a/examples/federated_analysis/histograms_example.svg b/examples/federated_analysis/histograms_example.svg new file mode 100644 index 00000000000..b2dfa9db5e3 --- /dev/null +++ b/examples/federated_analysis/histograms_example.svg @@ -0,0 +1 @@ +010020002M4M6M010020002M4M6M8M10M010020005M10M0100200010M20M30M0100200020M40M60Msite-4 (1345 of 1345 images)site-1 (3616 of 3616 images)site-2 (6012 of 6012 images)site-3 (10192 of 10192 images)Global (21165 of 21165 images)Histograms for 4 of 4 clientsImage IntensityCount \ No newline at end of file diff --git a/examples/federated_analysis/run_fl.py b/examples/federated_analysis/run_fl.py new file mode 100644 index 00000000000..b8490d925da --- /dev/null +++ b/examples/federated_analysis/run_fl.py @@ -0,0 +1,66 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import time + +from nvflare.fuel.hci.client.fl_admin_api_runner import FLAdminAPIRunner + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--run_number", type=int, default=100, help="FL run number to start at.") + parser.add_argument("--admin_dir", type=str, default="./admin/", help="Path to admin directory.") + parser.add_argument("--username", type=str, default="admin@nvidia.com", help="Admin username") + parser.add_argument("--app", type=str, default="fed_analysis", help="App to be deployed") + parser.add_argument("--port", type=int, default=8003, help="The admin server port") + parser.add_argument("--poc", action='store_true', help="Whether admin uses POC mode.") + parser.add_argument("--min_clients", type=int, default=8, help="Minimum number of clients.") + args = parser.parse_args() + + host = "" + port = args.port + + assert os.path.isdir(args.admin_dir), f"admin directory does not exist at {args.admin_dir}" + + # Set up certificate names and admin folders + upload_dir = os.path.join(args.admin_dir, "transfer") + if not os.path.isdir(upload_dir): + os.makedirs(upload_dir) + download_dir = os.path.join(args.admin_dir, "download") + if not os.path.isdir(download_dir): + os.makedirs(download_dir) + + run_number = args.run_number + + # Initialize the runner + runner = FLAdminAPIRunner( + host=host, + port=port, + username=args.username, + admin_dir=args.admin_dir, + poc=args.poc, + debug=False, + ) + + # Run + start = time.time() + runner.run(run_number, args.app, restart_all_first=False, shutdown_on_error=True, shutdown_at_end=True, + timeout=7200, min_clients=args.min_clients) # will time out if not completed in 2 hours + print("Total training time", time.time() - start) + + +if __name__ == "__main__": + main() diff --git a/examples/federated_analysis/run_poc.sh b/examples/federated_analysis/run_poc.sh new file mode 100755 index 00000000000..3a6ebb17ada --- /dev/null +++ b/examples/federated_analysis/run_poc.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# add current folder to PYTHONPATH +export PYTHONPATH="${PWD}" +echo "PYTHONPATH is ${PYTHONPATH}" + +algorithms_dir="${PWD}/configs" +servername="localhost" +workspace="workspaces/poc_workspace" +admin_username="admin" # default admin +site_pre="site-" + +n_clients=$1 +config=$2 +run=$3 + +if test -z "${n_clients}" || test -z "${config}" || test -z "${run}" +then + echo "Usage: ./run_poc.sh [n_clients] [config] [run], e.g. ./run_poc.sh 4 fed_analysis 1" + exit 1 +fi + +# start server +echo "STARTING SERVER" +export CUDA_VISIBLE_DEVICES=0 +./${workspace}/server/startup/start.sh ${servername} & +sleep 10 + +# start clients +echo "STARTING ${n_clients} CLIENTS" +for id in $(eval echo "{1..$n_clients}") +do + export CUDA_VISIBLE_DEVICES=0 + ./${workspace}/"${site_pre}${id}"/startup/start.sh ${servername} "${site_pre}${id}" & +done +sleep 10 + +# start training +echo "STARTING ANALYSIS" +python3 ./run_fl.py --port=8003 --admin_dir="./${workspace}/${admin_username}" \ + --run_number="${run}" --app="${algorithms_dir}/${config}" --min_clients="${n_clients}" --poc + +# sleep for FL system to shut down, so a new run can be started automatically +sleep 30 diff --git a/examples/federated_analysis/virtualenv/requirements.txt b/examples/federated_analysis/virtualenv/requirements.txt new file mode 100644 index 00000000000..863433dd015 --- /dev/null +++ b/examples/federated_analysis/virtualenv/requirements.txt @@ -0,0 +1,7 @@ +nvflare +numpy +itk-io +monai +pandas +kaleido +plotly \ No newline at end of file diff --git a/examples/federated_analysis/virtualenv/set_env.sh b/examples/federated_analysis/virtualenv/set_env.sh new file mode 100755 index 00000000000..adeca22ad41 --- /dev/null +++ b/examples/federated_analysis/virtualenv/set_env.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +export projectname='fedanalysis' +export projectpath="." + +python3 -m venv ${projectname} +source ${projectname}/bin/activate diff --git a/examples/federated_analysis/workspaces/.gitkeep b/examples/federated_analysis/workspaces/.gitkeep new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/examples/federated_analysis/workspaces/.gitkeep @@ -0,0 +1 @@ + diff --git a/nvflare/private/fed/app/server/server_train.py b/nvflare/private/fed/app/server/server_train.py index 9601b99ee42..1256985b7b8 100644 --- a/nvflare/private/fed/app/server/server_train.py +++ b/nvflare/private/fed/app/server/server_train.py @@ -66,7 +66,7 @@ def main(): # trainer = WorkFlowFactory().create_server_trainer(train_configs, envs) startup = os.path.join(args.workspace, "startup") conf = FLServerStarterConfiger( - app_root="startup", + app_root=startup, # wf_config_file_name="config_train.json", server_config_file_name=args.fed_server, # env_config_file_name="environment.json", diff --git a/test/app_testing/admin_controller.py b/test/app_testing/admin_controller.py index 66cd3d5d94a..9a52de20a5a 100644 --- a/test/app_testing/admin_controller.py +++ b/test/app_testing/admin_controller.py @@ -48,7 +48,6 @@ def initialize(self): success = False try: - # TODO:: login or login_with_password should return FLAdminAPIResponse to be consistent? response = None timeout = 100 start_time = time.time() diff --git a/test/app_testing/app_result_validator.py b/test/app_testing/app_result_validator.py index 8f7fe9a7d5d..3e24730ae66 100644 --- a/test/app_testing/app_result_validator.py +++ b/test/app_testing/app_result_validator.py @@ -13,12 +13,13 @@ # limitations under the License. import logging +from abc import ABC, abstractmethod -class AppResultValidator(object): +class AppResultValidator(ABC): def __init__(self): - super(AppResultValidator, self).__init__() self.logger = logging.getLogger("AppValidator") + @abstractmethod def validate_results(self, server_data, client_data, run_data) -> bool: pass diff --git a/test/app_testing/run_app_tests.sh b/test/app_testing/run_app_tests.sh index 7ca24059714..29b4c0ecc07 100755 --- a/test/app_testing/run_app_tests.sh +++ b/test/app_testing/run_app_tests.sh @@ -1,4 +1,5 @@ +#!/usr/bin/env bash pip install tensorflow -python test_runner.py --poc ../../nvflare/poc --n_clients 2 --yaml test_apps/test_simple.yml --app_path test_apps --cleanup \ No newline at end of file +python test_runner.py --poc ../../nvflare/poc --n_clients 2 --yaml test_simple.yml --app_path test_apps --cleanup diff --git a/test/app_testing/run_example_tests.sh b/test/app_testing/run_example_tests.sh index 6f2fc070f50..d8a4ccc0d76 100755 --- a/test/app_testing/run_example_tests.sh +++ b/test/app_testing/run_example_tests.sh @@ -1,4 +1,5 @@ +#!/usr/bin/env bash pip install tensorflow -python test_runner.py --poc ../../nvflare/poc --n_clients 2 --yaml test_apps/test_examples.yml --app_path ../../examples --cleanup \ No newline at end of file +python test_runner.py --poc ../../nvflare/poc --n_clients 2 --yaml test_examples.yml --app_path ../../examples --cleanup diff --git a/test/app_testing/site_launcher.py b/test/app_testing/site_launcher.py index 2fe9502b3d8..3406408b285 100644 --- a/test/app_testing/site_launcher.py +++ b/test/app_testing/site_launcher.py @@ -44,7 +44,7 @@ def __init__( """ This class sets up the test environment for a test. It will launch and keep track of servers and clients. """ - super(SiteLauncher, self).__init__() + super().__init__() self.original_poc_directory = poc_directory self.server_dir_name = server_dir_name @@ -57,7 +57,7 @@ def __init__( self.admin_api = None - self.logger = logging.getLogger("SiteRunner") + self.logger = logging.getLogger("SiteLauncher") # Create temporary poc directory if not os.path.exists(self.original_poc_directory): @@ -74,7 +74,6 @@ def __init__( def start_server(self): server_dir = os.path.join(self.poc_directory, self.server_dir_name) log_path = os.path.join(server_dir, "log.txt") - # log_file = open(log_path, 'w') new_env = os.environ.copy() # Create upload directory @@ -92,7 +91,6 @@ def start_server(self): stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) - # process = subprocess.Popen(shlex.split(command, " "), preexec_fn=os.setsid, env=new_env) print("Starting server ...") t = threading.Thread(target=process_logs, args=(log_path, process)) @@ -100,7 +98,6 @@ def start_server(self): self.server_properties["path"] = server_dir self.server_properties["process"] = process - # self.server_properties["log_file"] = log_file self.server_properties["log_path"] = log_path def start_clients(self, n=2): @@ -141,7 +138,6 @@ def start_clients(self, n=2): stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) - # process = subprocess.Popen(shlex.split(command, " "), preexec_fn=os.setsid, env=new_env) self.client_properties[client_id]["process"] = process print(f"Launched client {client_id} process.") @@ -223,9 +219,6 @@ def stop_all_clients(self): def stop_all_sites(self): self.stop_server() - def client_status(self, client_id): - pass - def finalize(self): print(f"Deleting temporary directory: {self.poc_directory}.") shutil.rmtree(self.poc_directory) diff --git a/test/app_testing/test_apps/test_examples.yml b/test/app_testing/test_apps/test_examples.yml deleted file mode 100644 index 700a34bf62a..00000000000 --- a/test/app_testing/test_apps/test_examples.yml +++ /dev/null @@ -1,20 +0,0 @@ - - - -tests: - - app_name: hello-pt - validators: - - test.app_testing.test_apps.validators.pt_model_validator.PTModelValidator - - app_name: hello-numpy-cross-val - validators: - - test.app_testing.test_apps.validators.cross_val_result_validator.CrossResultValidator - - test.app_testing.test_apps.validators.sag_result_validator.SAGResultValidator - - app_name: hello-numpy-sag - validators: - - test.app_testing.test_apps.validators.sag_result_validator.SAGResultValidator - - app_name: hello-tf2 - validators: - - test.app_testing.test_apps.validators.tf_model_validator.TFModelValidator - - app_name: hello-cyclic - validators: - - test.app_testing.test_apps.validators.tf_model_validator.TFModelValidator diff --git a/test/app_testing/test_apps/test_simple.yml b/test/app_testing/test_apps/test_simple.yml deleted file mode 100644 index ae313009bb7..00000000000 --- a/test/app_testing/test_apps/test_simple.yml +++ /dev/null @@ -1,37 +0,0 @@ - - - -tests: - - app_name: global_model_eval - validators: - - test.app_testing.test_apps.validators.global_model_eval_validator.GlobalModelEvalValidator - - test.app_testing.test_apps.validators.sag_result_validator.SAGResultValidator - - app_name: pt - validators: - - test.app_testing.test_apps.validators.pt_model_validator.PTModelValidator - - app_name: sag_weights_diff - validators: - - test.app_testing.test_apps.validators.sag_result_validator.SAGResultValidator - - app_name: cross_val_one_client - validators: - - test.app_testing.test_apps.validators.cross_val_single_client_validator.CrossValSingleClientValidator - - test.app_testing.test_apps.validators.sag_result_validator.SAGResultValidator - - app_name: cross_val - validators: - - test.app_testing.test_apps.validators.cross_val_result_validator.CrossResultValidator - - test.app_testing.test_apps.validators.sag_result_validator.SAGResultValidator - - app_name: sag - validators: - - test.app_testing.test_apps.validators.sag_result_validator.SAGResultValidator - - app_name: filters - validators: - - test.app_testing.test_apps.validators.filters_result_validator.FiltersResultValidator - - app_name: tb_streaming - validators: - - test.app_testing.test_apps.validators.tb_result_validator.TBResultValidator - - app_name: tf - validators: - - test.app_testing.test_apps.validators.tf_model_validator.TFModelValidator - - app_name: cyclic - validators: - - test.app_testing.test_apps.validators.tf_model_validator.TFModelValidator diff --git a/test/app_testing/test_examples.yml b/test/app_testing/test_examples.yml new file mode 100644 index 00000000000..65ca1f0f487 --- /dev/null +++ b/test/app_testing/test_examples.yml @@ -0,0 +1,20 @@ + + + +tests: + - app_name: hello-pt + validators: + - test.app_testing.validators.pt_model_validator.PTModelValidator + - app_name: hello-numpy-cross-val + validators: + - test.app_testing.validators.cross_val_result_validator.CrossResultValidator + - test.app_testing.validators.sag_result_validator.SAGResultValidator + - app_name: hello-numpy-sag + validators: + - test.app_testing.validators.sag_result_validator.SAGResultValidator + - app_name: hello-tf2 + validators: + - test.app_testing.validators.tf_model_validator.TFModelValidator + - app_name: hello-cyclic + validators: + - test.app_testing.validators.tf_model_validator.TFModelValidator diff --git a/test/app_testing/test_simple.yml b/test/app_testing/test_simple.yml new file mode 100644 index 00000000000..8c3b5075137 --- /dev/null +++ b/test/app_testing/test_simple.yml @@ -0,0 +1,37 @@ + + + +tests: + - app_name: global_model_eval + validators: + - test.app_testing.validators.global_model_eval_validator.GlobalModelEvalValidator + - test.app_testing.validators.sag_result_validator.SAGResultValidator + - app_name: pt + validators: + - test.app_testing.validators.pt_model_validator.PTModelValidator + - app_name: sag_weights_diff + validators: + - test.app_testing.validators.sag_result_validator.SAGResultValidator + - app_name: cross_val_one_client + validators: + - test.app_testing.validators.cross_val_single_client_validator.CrossValSingleClientValidator + - test.app_testing.validators.sag_result_validator.SAGResultValidator + - app_name: cross_val + validators: + - test.app_testing.validators.cross_val_result_validator.CrossResultValidator + - test.app_testing.validators.sag_result_validator.SAGResultValidator + - app_name: sag + validators: + - test.app_testing.validators.sag_result_validator.SAGResultValidator + - app_name: filters + validators: + - test.app_testing.validators.filters_result_validator.FiltersResultValidator + - app_name: tb_streaming + validators: + - test.app_testing.validators.tb_result_validator.TBResultValidator + - app_name: tf + validators: + - test.app_testing.validators.tf_model_validator.TFModelValidator + - app_name: cyclic + validators: + - test.app_testing.validators.tf_model_validator.TFModelValidator diff --git a/test/app_testing/test_apps/validators/cross_val_result_validator.py b/test/app_testing/validators/cross_val_result_validator.py similarity index 100% rename from test/app_testing/test_apps/validators/cross_val_result_validator.py rename to test/app_testing/validators/cross_val_result_validator.py diff --git a/test/app_testing/test_apps/validators/cross_val_single_client_validator.py b/test/app_testing/validators/cross_val_single_client_validator.py similarity index 97% rename from test/app_testing/test_apps/validators/cross_val_single_client_validator.py rename to test/app_testing/validators/cross_val_single_client_validator.py index d41899c19c1..642f98f6e92 100644 --- a/test/app_testing/test_apps/validators/cross_val_single_client_validator.py +++ b/test/app_testing/validators/cross_val_single_client_validator.py @@ -13,11 +13,10 @@ # limitations under the License. import os - from test.app_testing.app_result_validator import AppResultValidator -def check_cross_validation_result(server_data, client_data, run_data): +def check_cross_validation_result(server_data, run_data): run_number = run_data["run_number"] server_dir = server_data["server_path"] @@ -79,7 +78,7 @@ def __init__(self): def validate_results(self, server_data, client_data, run_data) -> bool: - cross_val_result = check_cross_validation_result(server_data, client_data, run_data) + cross_val_result = check_cross_validation_result(server_data, run_data) print(f"CrossVal Result: {cross_val_result}") diff --git a/test/app_testing/test_apps/validators/filters_result_validator.py b/test/app_testing/validators/filters_result_validator.py similarity index 100% rename from test/app_testing/test_apps/validators/filters_result_validator.py rename to test/app_testing/validators/filters_result_validator.py diff --git a/test/app_testing/test_apps/validators/global_model_eval_validator.py b/test/app_testing/validators/global_model_eval_validator.py similarity index 100% rename from test/app_testing/test_apps/validators/global_model_eval_validator.py rename to test/app_testing/validators/global_model_eval_validator.py diff --git a/test/app_testing/test_apps/validators/pt_model_validator.py b/test/app_testing/validators/pt_model_validator.py similarity index 99% rename from test/app_testing/test_apps/validators/pt_model_validator.py rename to test/app_testing/validators/pt_model_validator.py index 8b8843c169d..95de8f83c4a 100644 --- a/test/app_testing/test_apps/validators/pt_model_validator.py +++ b/test/app_testing/validators/pt_model_validator.py @@ -14,11 +14,10 @@ import os from collections import OrderedDict +from test.app_testing.app_result_validator import AppResultValidator import numpy as np -from test.app_testing.app_result_validator import AppResultValidator - def check_pt_results(server_data, client_data, run_data): run_number = run_data["run_number"] diff --git a/test/app_testing/test_apps/validators/sag_result_validator.py b/test/app_testing/validators/sag_result_validator.py similarity index 100% rename from test/app_testing/test_apps/validators/sag_result_validator.py rename to test/app_testing/validators/sag_result_validator.py diff --git a/test/app_testing/test_apps/validators/tb_result_validator.py b/test/app_testing/validators/tb_result_validator.py similarity index 100% rename from test/app_testing/test_apps/validators/tb_result_validator.py rename to test/app_testing/validators/tb_result_validator.py diff --git a/test/app_testing/test_apps/validators/tf_model_validator.py b/test/app_testing/validators/tf_model_validator.py similarity index 99% rename from test/app_testing/test_apps/validators/tf_model_validator.py rename to test/app_testing/validators/tf_model_validator.py index 49a00b0c36a..ad0c4cd709d 100644 --- a/test/app_testing/test_apps/validators/tf_model_validator.py +++ b/test/app_testing/validators/tf_model_validator.py @@ -13,9 +13,8 @@ # limitations under the License. import os -from test.app_testing.app_result_validator import AppResultValidator - import pickle +from test.app_testing.app_result_validator import AppResultValidator def check_tf_results(server_data, client_data, run_data): diff --git a/test/test_jsonscaner.py b/test/test_jsonscaner.py deleted file mode 100644 index 41968a38ddf..00000000000 --- a/test/test_jsonscaner.py +++ /dev/null @@ -1,252 +0,0 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import unittest - -from nvflare.fuel.utils.json_scanner import JsonObjectProcessor, JsonScanner, Node - - -class _TestJsonProcessor(JsonObjectProcessor): - def __init__(self): - JsonObjectProcessor.__init__(self) - - def process_element(self, node: Node): - pats = [ - r".\.pre_transforms\.#[0-9]+$", - r"^train\.model\.name$", - r".\.search\.#[0-9]+$", - r".\.pre_transforms\.#[0-9]+\.args$", - ] - path = node.path() - print("ENTER Level: {}; Key: {}; Pos: {}; Path: {}".format(node.level, node.key, node.position, path)) - for p in pats: - x = re.search(p, path) - if x: - print("\t {} matches {}".format(path, p)) - - node.exit_cb = self.post_process_element - - def post_process_element(self, node: Node): - path = node.path() - print("EXIT Level: {}; Key: {}; Pos: {}; Path: {}".format(node.level, node.key, node.position, path)) - - -def _test(): - import json - - test_json = """ - { - "learning_rate": 1e-4, - "lr_search" : [1e-4, 2e-3], - "train": { - "model": { - "name": "SegAhnet", - "args": { - "num_classes": 2, - "if_use_psp": false, - "pretrain_weight_name": "{PRETRAIN_WEIGHTS_FILE}", - "plane": "z", - "final_activation": "softmax", - "n_spatial_dim": 3 - }, - "search": [ - { - "type": "float", - "args": ["num_classes"], - "targets": [1,3], - "domain": "net" - }, - { - "type": "float", - "args": ["n_spatial_dim"], - "targets": [2,5], - "domain": "net" - }, - { - "type": "enum", - "args": ["n_spatial_dim", "num_classes"], - "targets": [[2,3],[3,4],[5,1]], - "domain": "net" - }, - { - "type": "enum", - "args": ["n_spatial_dim"], - "targets": [[2],[3],[6],[12]], - "domain": "net" - } - ] - }, - "pre_transforms": [ - { - "name": "LoadNifti", - "args": { - "fields": [ - "image", - "label" - ] - } - }, - { - "name": "ConvertToChannelsFirst", - "args": { - "fields": [ - "image", - "label" - ] - } - }, - { - "name": "ScaleIntensityRange", - "args": { - "fields": "image", - "a_min": -57, - "a_max": 164, - "b_min": 0.0, - "b_max": 1.0, - "clip": true - } - }, - { - "name": "FastCropByPosNegRatio", - "args": { - "size": [ - 96, - 96, - 96 - ], - "fields": "image", - "label_field": "label", - "pos": 1, - "neg": 1, - "batch_size": 3 - }, - "search": [ - { - "domain": "transform", - "type": "enum", - "args": ["size"], - "targets": [[[32, 32, 32]], [[64, 64, 64]], [[128, 128, 128]]] - }, - { - "domain": "transform", - "type": "enum", - "args": ["batch_size"], - "targets": [[3], [4], [8], [10]] - } - ] - }, - { - "name": "RandomAxisFlip", - "args": { - "fields": [ - "image", - "label" - ], - "probability": 0.0 - }, - "search": [ - { - "domain": "transform", - "type": "float", - "args": ["probability#p"], - "targets": [0.0, 1.0] - }, - { - "domain": "transform", - "args": "DISABLED" - } - ] - }, - { - "name": "RandomRotate3D", - "args": { - "fields": [ - "image", - "label" - ], - "probability": 0.0 - } - }, - { - "name": "ScaleIntensityOscillation", - "args": { - "fields": "image", - "magnitude": 0.10 - } - }, - { - "name": "LoadNifti", - "args": { - "fields": [ - "image", - "label" - ] - } - }, - { - "name": "LoadNifti", - "args": { - "fields": [ - "image", - "label" - ] - } - }, - { - "name": "LoadNifti", - "args": { - "fields": [ - "image", - "label" - ] - } - }, - { - "name": "LoadNifti", - "args": { - "fields": [ - "image", - "label" - ] - } - }, - { - "name": "RandomAxisFlip", - "args": { - "fields": [ - "image", - "label" - ], - "probability": 0.0 - }, - "search": [ - { - "domain": "transform", - "type": "float", - "args": ["probability#p"], - "targets": [0.0, 1.0] - }, - { - "domain": "transform", - "args": "DISABLED" - } - ] - } - ] - } } - """ - train_config = json.loads(test_json) - scanner = JsonScanner(train_config, "test") - scanner.scan(_TestJsonProcessor()) diff --git a/test/controller/__init__.py b/test/unit_test/__init__.py similarity index 100% rename from test/controller/__init__.py rename to test/unit_test/__init__.py diff --git a/test/test_analytix.py b/test/unit_test/apis/test_analytix.py similarity index 100% rename from test/test_analytix.py rename to test/unit_test/apis/test_analytix.py diff --git a/test/test_fl_context.py b/test/unit_test/apis/test_fl_context.py similarity index 100% rename from test/test_fl_context.py rename to test/unit_test/apis/test_fl_context.py diff --git a/test/test_format_check.py b/test/unit_test/apis/test_format_check.py similarity index 100% rename from test/test_format_check.py rename to test/unit_test/apis/test_format_check.py diff --git a/test/test_aggregator.py b/test/unit_test/app_common/test_accumulate_weighted_aggregator.py similarity index 100% rename from test/test_aggregator.py rename to test/unit_test/app_common/test_accumulate_weighted_aggregator.py diff --git a/test/test_exclude_vars.py b/test/unit_test/app_common/test_exclude_vars.py similarity index 100% rename from test/test_exclude_vars.py rename to test/unit_test/app_common/test_exclude_vars.py diff --git a/test/test_intime_aggregator.py b/test/unit_test/app_common/test_in_time_accumulate_weighted_aggregator.py similarity index 100% rename from test/test_intime_aggregator.py rename to test/unit_test/app_common/test_in_time_accumulate_weighted_aggregator.py diff --git a/test/test_model_selection.py b/test/unit_test/app_common/test_in_time_model_selection_handler.py similarity index 94% rename from test/test_model_selection.py rename to test/unit_test/app_common/test_in_time_model_selection_handler.py index 0580b2a2b5f..030d98cde25 100644 --- a/test/test_model_selection.py +++ b/test/unit_test/app_common/test_in_time_model_selection_handler.py @@ -42,29 +42,29 @@ def fire_event(self, event_type: str, fl_ctx: FLContext): return True -class TestModelSelectionHandler: +class TestInTimeModelSelectionHandler: @pytest.mark.parametrize("mshandler", [IntimeModelSelectionHandler]) @pytest.mark.parametrize( "initial,received,expected", [ - [ + ( 1, { "client1": {"weight": 0.5, "iter_number": 1, "metric": 10}, }, True, - ], - [ + ), + ( 1, { "client1": {"weight": 0.5, "iter_number": 1, "metric": 1}, "client2": {"weight": 0.5, "iter_number": 1, "metric": 0.2}, }, False, - ], + ), ], ) - def test_model_section(self, mshandler, initial, received, expected): + def test_model_selection(self, mshandler, initial, received, expected): aggregation_weights = {k: v["weight"] for k, v in received.items()} handler = mshandler(aggregation_weights=aggregation_weights) handler.best_val_metric = initial diff --git a/test/test_streaming.py b/test/unit_test/app_common/test_streaming.py similarity index 87% rename from test/test_streaming.py rename to test/unit_test/app_common/test_streaming.py index 76b9d6c0fdb..14f66222886 100644 --- a/test/test_streaming.py +++ b/test/unit_test/app_common/test_streaming.py @@ -18,7 +18,7 @@ from nvflare.apis.dxo import DXO, DataKind from nvflare.apis.fl_component import FLComponent from nvflare.apis.fl_context import FLContext -from nvflare.app_common.widgets.streaming import send_analytic_dxo, create_analytic_dxo +from nvflare.app_common.widgets.streaming import create_analytic_dxo, send_analytic_dxo INVALID_TEST_CASES = [ (list(), dict(), FLContext(), TypeError, f"expect comp to be an instance of FLComponent, but got {type(list())}"), @@ -34,8 +34,14 @@ INVALID_WRITE_TEST_CASES = [ (list(), 1.0, AnalyticsDataType.SCALAR, TypeError, f"expect tag to be an instance of str, but got {type(list())}"), - ("tag", list(), AnalyticsDataType.SCALAR, TypeError, f"expect value to be an instance of float, but got {type(list())}"), - (list(), 1.0, AnalyticsDataType.SCALARS, TypeError, f"expect tag to be an instance of str, but got {type(list())}"), + ( + "tag", + list(), + AnalyticsDataType.SCALAR, + TypeError, + f"expect value to be an instance of float, but got {type(list())}", + ), + (list(), 1.0, AnalyticsDataType.SCALARS, TypeError, f"expect tag to be an instance of str, but got {type(list())}"), ("tag", 1.0, AnalyticsDataType.SCALARS, TypeError, f"expect value to be an instance of dict, but got {type(1.0)}"), (list(), 1.0, AnalyticsDataType.TEXT, TypeError, f"expect tag to be an instance of str, but got {type(list())}"), ("tag", 1.0, AnalyticsDataType.TEXT, TypeError, f"expect value to be an instance of str, but got {type(1.0)}"), diff --git a/test/unit_test/controller/__init__.py b/test/unit_test/controller/__init__.py new file mode 100644 index 00000000000..2b8f6c7e874 --- /dev/null +++ b/test/unit_test/controller/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/controller/controller_test.py b/test/unit_test/controller/controller_test.py similarity index 100% rename from test/controller/controller_test.py rename to test/unit_test/controller/controller_test.py diff --git a/test/controller/test_basic.py b/test/unit_test/controller/test_basic.py similarity index 100% rename from test/controller/test_basic.py rename to test/unit_test/controller/test_basic.py diff --git a/test/controller/test_broadcast_behavior.py b/test/unit_test/controller/test_broadcast_behavior.py similarity index 100% rename from test/controller/test_broadcast_behavior.py rename to test/unit_test/controller/test_broadcast_behavior.py diff --git a/test/controller/test_callback.py b/test/unit_test/controller/test_callback.py similarity index 100% rename from test/controller/test_callback.py rename to test/unit_test/controller/test_callback.py diff --git a/test/controller/test_invalid_input.py b/test/unit_test/controller/test_invalid_input.py similarity index 99% rename from test/controller/test_invalid_input.py rename to test/unit_test/controller/test_invalid_input.py index 1383cfba0cf..4e1fa61cd78 100644 --- a/test/controller/test_invalid_input.py +++ b/test/unit_test/controller/test_invalid_input.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest import re +import pytest + from nvflare.apis.controller_spec import SendOrder from nvflare.apis.fl_context import FLContext from nvflare.apis.shareable import Shareable diff --git a/test/controller/test_relay_behavior.py b/test/unit_test/controller/test_relay_behavior.py similarity index 100% rename from test/controller/test_relay_behavior.py rename to test/unit_test/controller/test_relay_behavior.py diff --git a/test/controller/test_send_bahavior.py b/test/unit_test/controller/test_send_bahavior.py similarity index 100% rename from test/controller/test_send_bahavior.py rename to test/unit_test/controller/test_send_bahavior.py diff --git a/test/controller/test_task.py b/test/unit_test/controller/test_task.py similarity index 100% rename from test/controller/test_task.py rename to test/unit_test/controller/test_task.py diff --git a/test/controller/test_task_mgmt.py b/test/unit_test/controller/test_task_mgmt.py similarity index 100% rename from test/controller/test_task_mgmt.py rename to test/unit_test/controller/test_task_mgmt.py diff --git a/test/unit_test/fuel/test_json_scanner.py b/test/unit_test/fuel/test_json_scanner.py new file mode 100644 index 00000000000..878e61d2698 --- /dev/null +++ b/test/unit_test/fuel/test_json_scanner.py @@ -0,0 +1,250 @@ +# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import re + +from nvflare.fuel.utils.json_scanner import JsonObjectProcessor, JsonScanner, Node + +test_json = """ +{ + "learning_rate": 1e-4, + "lr_search" : [1e-4, 2e-3], + "train": { + "model": { + "name": "SegAhnet", + "args": { + "num_classes": 2, + "if_use_psp": false, + "pretrain_weight_name": "{PRETRAIN_WEIGHTS_FILE}", + "plane": "z", + "final_activation": "softmax", + "n_spatial_dim": 3 + }, + "search": [ + { + "type": "float", + "args": ["num_classes"], + "targets": [1,3], + "domain": "net" + }, + { + "type": "float", + "args": ["n_spatial_dim"], + "targets": [2,5], + "domain": "net" + }, + { + "type": "enum", + "args": ["n_spatial_dim", "num_classes"], + "targets": [[2,3],[3,4],[5,1]], + "domain": "net" + }, + { + "type": "enum", + "args": ["n_spatial_dim"], + "targets": [[2],[3],[6],[12]], + "domain": "net" + } + ] + }, + "pre_transforms": [ + { + "name": "LoadNifti", + "args": { + "fields": [ + "image", + "label" + ] + } + }, + { + "name": "ConvertToChannelsFirst", + "args": { + "fields": [ + "image", + "label" + ] + } + }, + { + "name": "ScaleIntensityRange", + "args": { + "fields": "image", + "a_min": -57, + "a_max": 164, + "b_min": 0.0, + "b_max": 1.0, + "clip": true + } + }, + { + "name": "FastCropByPosNegRatio", + "args": { + "size": [ + 96, + 96, + 96 + ], + "fields": "image", + "label_field": "label", + "pos": 1, + "neg": 1, + "batch_size": 3 + }, + "search": [ + { + "domain": "transform", + "type": "enum", + "args": ["size"], + "targets": [[[32, 32, 32]], [[64, 64, 64]], [[128, 128, 128]]] + }, + { + "domain": "transform", + "type": "enum", + "args": ["batch_size"], + "targets": [[3], [4], [8], [10]] + } + ] + }, + { + "name": "RandomAxisFlip", + "args": { + "fields": [ + "image", + "label" + ], + "probability": 0.0 + }, + "search": [ + { + "domain": "transform", + "type": "float", + "args": ["probability#p"], + "targets": [0.0, 1.0] + }, + { + "domain": "transform", + "args": "DISABLED" + } + ] + }, + { + "name": "RandomRotate3D", + "args": { + "fields": [ + "image", + "label" + ], + "probability": 0.0 + } + }, + { + "name": "ScaleIntensityOscillation", + "args": { + "fields": "image", + "magnitude": 0.10 + } + }, + { + "name": "LoadNifti", + "args": { + "fields": [ + "image", + "label" + ] + } + }, + { + "name": "LoadNifti", + "args": { + "fields": [ + "image", + "label" + ] + } + }, + { + "name": "LoadNifti", + "args": { + "fields": [ + "image", + "label" + ] + } + }, + { + "name": "LoadNifti", + "args": { + "fields": [ + "image", + "label" + ] + } + }, + { + "name": "RandomAxisFlip", + "args": { + "fields": [ + "image", + "label" + ], + "probability": 0.0 + }, + "search": [ + { + "domain": "transform", + "type": "float", + "args": ["probability#p"], + "targets": [0.0, 1.0] + }, + { + "domain": "transform", + "args": "DISABLED" + } + ] + } +] +} } +""" +TRAIN_CONFIG = json.loads(test_json) + + +def _post_process_element(node: Node): + path = node.path() + print("EXIT Level: {}; Key: {}; Pos: {}; Path: {}".format(node.level, node.key, node.position, path)) + + +class _TestJsonProcessor(JsonObjectProcessor): + def process_element(self, node: Node): + pats = [ + r".\.pre_transforms\.#[0-9]+$", + r"^train\.model\.name$", + r".\.search\.#[0-9]+$", + r".\.pre_transforms\.#[0-9]+\.args$", + ] + path = node.path() + print("ENTER Level: {}; Key: {}; Pos: {}; Path: {}".format(node.level, node.key, node.position, path)) + for p in pats: + x = re.search(p, path) + if x: + print("\t {} matches {}".format(path, p)) + + node.exit_cb = _post_process_element + + +class TestJsonScanner: + def test_scan(self): + scanner = JsonScanner(TRAIN_CONFIG, "test") + scanner.scan(_TestJsonProcessor()) diff --git a/test/test_security_utils.py b/test/unit_test/fuel/test_security_utils.py similarity index 100% rename from test/test_security_utils.py rename to test/unit_test/fuel/test_security_utils.py diff --git a/test/test_messageproto.py b/test/unit_test/private/test_messageproto.py similarity index 100% rename from test/test_messageproto.py rename to test/unit_test/private/test_messageproto.py diff --git a/test/utils.py b/test/unit_test/utils.py similarity index 100% rename from test/utils.py rename to test/unit_test/utils.py