Maintained by Difan Deng and Marius Lindauer.
The following list considers papers related to neural architecture search. It is by no means complete. If you miss a paper on the list, please let us know.
Please note that although NAS methods steadily improve, the quality of empirical evaluations in this field are still lagging behind compared to other areas in machine learning, AI and optimization. We would therefore like to share some best practices for empirical evaluations of NAS methods, which we believe will facilitate sustained and measurable progress in the field. If you are interested in a teaser, please read our blog post or directly jump to our checklist.
Transformers have gained increasing popularity in different domains. For a comprehensive list of papers focusing on Neural Architecture Search for Transformer-Based spaces, the awesome-transformer-search repo is all you need.
2022
1646.
Hu, Zhonggang; Bao, Wenxing; Qu, Kewen; Liang, Hongbo
Image-based neural architecture automatic search method for hyperspectral image classification Journal Article
In: J. of Applied Remote Sensing, 16 , 2022.
@article{Hu2022,
title = {Image-based neural architecture automatic search method for hyperspectral image classification},
author = {Zhonggang Hu and Wenxing Bao and Kewen Qu and Hongbo Liang
},
url = {https://www.spiedigitallibrary.org/journals/journal-of-applied-remote-sensing/volume-16/issue-01/016501/Image-based-neural-architecture-automatic-search-method-for-hyperspectral-image/10.1117/1.JRS.16.016501.full?tab=ArticleLinkFigureTable},
year = {2022},
date = {2022-01-06},
journal = {J. of Applied Remote Sensing},
volume = {16},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
1645.
Diao, Huabin; Li, Gongyan; Xu, Shaoyun; Hao, Yuexing
PA-NAS: Partial operation activation for memory-efficient architecture search Journal Article
In: Applied Intelligence, 2022.
@article{DiaoPANAS2022,
title = {PA-NAS: Partial operation activation for memory-efficient architecture search},
author = {Huabin Diao and Gongyan Li and Shaoyun Xu and Yuexing Hao},
url = {https://link.springer.com/article/10.1007/s10489-021-02961-8},
year = {2022},
date = {2022-01-05},
urldate = {2022-01-05},
journal = {Applied Intelligence},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
1644.
Klos, Andreas; Rosenbaum, Marius; Schiffmann, Wolfram
Neural Architecture Search based on Genetic Algorithm and Deployed in a Bare-Metal Kubernetes Cluster Journal Article
In: International Journal of Networking and Computing, 12 (1), 2022.
@article{Klos2022,
title = {Neural Architecture Search based on Genetic Algorithm and Deployed in a Bare-Metal Kubernetes Cluster},
author = {Andreas Klos and Marius Rosenbaum and Wolfram Schiffmann},
url = {http://www.ijnc.org/index.php/ijnc/article/view/276},
year = {2022},
date = {2022-01-03},
urldate = {2022-01-03},
journal = {International Journal of Networking and Computing},
volume = {12},
number = {1},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
1643.
Jiang, Chunheng; Pedapati, Tejaswini; Chen, Pin-Yu; Sun, Yizhou; Gao, Jianxi
Neural Capacitance: A New Perspective of Neural Network Selection via Edge Dynamics Technical Report
, 2022.
@techreport{DBLP:journals/corr/abs-2201-04194,
title = {Neural Capacitance: A New Perspective of Neural Network Selection via Edge Dynamics},
author = {Chunheng Jiang and Tejaswini Pedapati and Pin-Yu Chen and Yizhou Sun and Jianxi Gao},
url = {https://arxiv.org/abs/2201.04194},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {CoRR},
volume = {abs/2201.04194},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
1642.
Durbha, Krishna Srikar; Amuru, Saidhiraj
AutoML Models for Wireless Signals Classification and their effectiveness against Adversarial Attacks Inproceedings
In: 2022 14th International Conference on COMmunication Systems NETworkS (COMSNETS), pp. 265-269, 2022.
@inproceedings{9668448,
title = {AutoML Models for Wireless Signals Classification and their effectiveness against Adversarial Attacks},
author = {Krishna Srikar Durbha and Saidhiraj Amuru},
url = {https://ieeexplore.ieee.org/abstract/document/9668448},
doi = {10.1109/COMSNETS53615.2022.9668448},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
booktitle = {2022 14th International Conference on COMmunication Systems NETworkS (COMSNETS)},
pages = {265-269},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
1641.
Pochelu, Pierrick; Petiton, Serge G.; Conche, Bruno
A Deep Neural Networks Ensemble Workflow from Hyperparameter Search to Inference Leveraging GPU Clusters Inproceedings
In: International Conference on High Performance Computing in Asia-Pacific Region, pp. 61–71, Association for Computing Machinery, Virtual Event, Japan, 2022, ISBN: 9781450384988.
@inproceedings{10.1145/3492805.3492819,
title = {A Deep Neural Networks Ensemble Workflow from Hyperparameter Search to Inference Leveraging GPU Clusters},
author = {Pierrick Pochelu and Serge G. Petiton and Bruno Conche},
url = {https://doi.org/10.1145/3492805.3492819},
doi = {10.1145/3492805.3492819},
isbn = {9781450384988},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
booktitle = {International Conference on High Performance Computing in Asia-Pacific Region},
pages = {61–71},
publisher = {Association for Computing Machinery},
address = {Virtual Event, Japan},
series = {HPCAsia2022},
abstract = {Automated Machine Learning with ensembling (or AutoML with ensembling) seeks to automatically build ensembles of Deep Neural Networks (DNNs) to achieve qualitative predictions. Ensemble of DNNs are well known to avoid over-fitting but they are memory and time consuming approaches. Therefore, an ideal AutoML would produce in one single run time different ensembles regarding accuracy and inference speed. While previous works on AutoML focus to search for the best model to maximize its generalization ability, we rather propose a new AutoML to build a larger library of accurate and diverse individual models to then construct ensembles. First, our extensive benchmarks show asynchronous Hyperband is an efficient and robust way to build a large number of diverse models to combine them. Then, a new ensemble selection method based on a multi-objective greedy algorithm is proposed to generate accurate ensembles by controlling their computing cost. Finally, we propose a novel algorithm to optimize the inference of the DNNs ensemble in a GPU cluster based on allocation optimization. The produced AutoML with ensemble method shows robust results on two datasets using efficiently GPU clusters during both the training phase and the inference phase.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Automated Machine Learning with ensembling (or AutoML with ensembling) seeks to automatically build ensembles of Deep Neural Networks (DNNs) to achieve qualitative predictions. Ensemble of DNNs are well known to avoid over-fitting but they are memory and time consuming approaches. Therefore, an ideal AutoML would produce in one single run time different ensembles regarding accuracy and inference speed. While previous works on AutoML focus to search for the best model to maximize its generalization ability, we rather propose a new AutoML to build a larger library of accurate and diverse individual models to then construct ensembles. First, our extensive benchmarks show asynchronous Hyperband is an efficient and robust way to build a large number of diverse models to combine them. Then, a new ensemble selection method based on a multi-objective greedy algorithm is proposed to generate accurate ensembles by controlling their computing cost. Finally, we propose a novel algorithm to optimize the inference of the DNNs ensemble in a GPU cluster based on allocation optimization. The produced AutoML with ensemble method shows robust results on two datasets using efficiently GPU clusters during both the training phase and the inference phase.
1640.
Chen, Yi-Ping; Tsai, Chun-Wei
An Effective Neural Architecture Optimization Algorithm for CNN Based on Search Economics Inproceedings
In: Proceedings of the 2021 ACM International Conference on Intelligent Computing and Its Emerging Applications, pp. 40–45, Association for Computing Machinery, Jinan, China, 2022, ISBN: 9781450391603.
@inproceedings{10.1145/3491396.3506505,
title = {An Effective Neural Architecture Optimization Algorithm for CNN Based on Search Economics},
author = {Yi-Ping Chen and Chun-Wei Tsai},
url = {https://doi.org/10.1145/3491396.3506505},
doi = {10.1145/3491396.3506505},
isbn = {9781450391603},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
booktitle = {Proceedings of the 2021 ACM International Conference on Intelligent Computing and Its Emerging Applications},
pages = {40–45},
publisher = {Association for Computing Machinery},
address = {Jinan, China},
series = {ACM ICEA '21},
abstract = {Developing a good convolutional neural network (CNN) architecture manually by trial and error is typically extremely time-consuming and requires a lot of effort. That is why several recent studies have attempted to develop ways to automatically construct a suitable CNN architecture. In this study, an effective neural architecture search (NAS) algorithm based on a novel metaheuristic algorithm, search economics (SE), is presented for CNN to improve the accuracy of image classification. The basic idea of the proposed algorithm is to use the "expected value" instead of the öbjective value" to evaluate a set of searched solutions, i.e., neural architectures in this case. As such, the searches of the proposed algorithm will trend to high potential regions in the solution space. Simulation results show that the proposed algorithm outperforms genetic algorithm-based NAS algorithm in terms of the accuracy, especially for complex image classification problems.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Developing a good convolutional neural network (CNN) architecture manually by trial and error is typically extremely time-consuming and requires a lot of effort. That is why several recent studies have attempted to develop ways to automatically construct a suitable CNN architecture. In this study, an effective neural architecture search (NAS) algorithm based on a novel metaheuristic algorithm, search economics (SE), is presented for CNN to improve the accuracy of image classification. The basic idea of the proposed algorithm is to use the "expected value" instead of the öbjective value" to evaluate a set of searched solutions, i.e., neural architectures in this case. As such, the searches of the proposed algorithm will trend to high potential regions in the solution space. Simulation results show that the proposed algorithm outperforms genetic algorithm-based NAS algorithm in terms of the accuracy, especially for complex image classification problems.
1639.
Wu, Meng-Ting; Lin, Hung-I; Tsai, Chun-Wei
A Training-Free Genetic Neural Architecture Search Inproceedings
In: Proceedings of the 2021 ACM International Conference on Intelligent Computing and Its Emerging Applications, pp. 65–70, Association for Computing Machinery, Jinan, China, 2022, ISBN: 9781450391603.
@inproceedings{10.1145/3491396.3506510,
title = {A Training-Free Genetic Neural Architecture Search},
author = {Meng-Ting Wu and Hung-I Lin and Chun-Wei Tsai},
url = {https://doi.org/10.1145/3491396.3506510},
doi = {10.1145/3491396.3506510},
isbn = {9781450391603},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
booktitle = {Proceedings of the 2021 ACM International Conference on Intelligent Computing and Its Emerging Applications},
pages = {65–70},
publisher = {Association for Computing Machinery},
address = {Jinan, China},
series = {ACM ICEA '21},
abstract = {The so-called neural architecture search (NAS) provides an alternative way to construct a "good neural architecture," which would normally outperform hand-made architectures, for solving complex problems without domain knowledge. However, a critical issue for most of the NAS techniques is in that it is computationally very expensive because several complete/partial training processes are involved in evaluating the goodness of a neural architecture during the process of NAS. To mitigate this problem for evaluating a single neural architecture found by the search algorithm of NAS, we present an efficient NAS in this study, called genetic algorithm and noise immunity for neural architecture search without training (GA-NINASWOT). The genetic algorithm (GA) in the proposed algorithm is used to search for high potential neural architectures while a modified scoring method based on the neural architecture search without training (NASWOT) is used to replace the training process of each neural architecture found by the GA for measuring its quality. To evaluate the performance of GA-NINASWOT, we compared it with several state-of-the-art NAS techniques, which include weight-sharing methods, non-weight-sharing methods, and NASWOT. Simulation results show that GA-NINASWOT outperforms all the other state-of-the-art weight-sharing methods and NASWOT compared in this study in terms of the accuracy and computational time. Moreover, GA-NINASWOT gives a result that is comparable to those found by the non-weight-sharing methods while reducing 99% of the search time.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
The so-called neural architecture search (NAS) provides an alternative way to construct a "good neural architecture," which would normally outperform hand-made architectures, for solving complex problems without domain knowledge. However, a critical issue for most of the NAS techniques is in that it is computationally very expensive because several complete/partial training processes are involved in evaluating the goodness of a neural architecture during the process of NAS. To mitigate this problem for evaluating a single neural architecture found by the search algorithm of NAS, we present an efficient NAS in this study, called genetic algorithm and noise immunity for neural architecture search without training (GA-NINASWOT). The genetic algorithm (GA) in the proposed algorithm is used to search for high potential neural architectures while a modified scoring method based on the neural architecture search without training (NASWOT) is used to replace the training process of each neural architecture found by the GA for measuring its quality. To evaluate the performance of GA-NINASWOT, we compared it with several state-of-the-art NAS techniques, which include weight-sharing methods, non-weight-sharing methods, and NASWOT. Simulation results show that GA-NINASWOT outperforms all the other state-of-the-art weight-sharing methods and NASWOT compared in this study in terms of the accuracy and computational time. Moreover, GA-NINASWOT gives a result that is comparable to those found by the non-weight-sharing methods while reducing 99% of the search time.
1638.
Yipeng, Lv; Wenbing, Lv; Kaixuan, Han; Wentao, Tao; Ling, Zheng; Shizhuang, Weng; Lingsheng, Huang
In: Food Control, pp. 108819, 2022, ISSN: 0956-7135.
@article{YIPENG2022108819,
title = {Determination of wheat kernels damaged by Fusarium head blight using monochromatic images of effective wavelengths from hyperspectral imaging coupled with an architecture self-search deep network},
author = {Lv Yipeng and Lv Wenbing and Han Kaixuan and Tao Wentao and Zheng Ling and Weng Shizhuang and Huang Lingsheng},
url = {https://www.sciencedirect.com/science/article/pii/S0956713522000123},
doi = {https://doi.org/10.1016/j.foodcont.2022.108819},
issn = {0956-7135},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {Food Control},
pages = {108819},
abstract = {Wheat kernels damaged by Fusarium head blight (FHB) lose moisture, protein and starch and carry dangerous toxins. Classification of degree of damage can be helpful to customise the use of wheat kernels, reduce financial losses and ensure grain safety. In this study, hyperspectral imaging (HSI) and deep learning network were explored to determine sound, mildly, moderately and severely damaged wheat kernels. Effective wavelengths (EWs) were selected from the reflectance spectroscopy of HSI images by ReliefF, uninformative variable elimination, random frog and shuffled frog leaping algorithm, and the monochromatic images of different combinations of EWs were adopted to develop the classification models combined with an architecture self-search deep network (ASSDN). ASSDN and the images composed of 941, 876 and 732 nm achieved the best determination with average accuracy of 100% and 98.31% in the training and prediction sets, respectively, outperforming other images or methods. And the average area under the curve of 0.9985 indicated its excellent robustness. Using images of sporadic wavelengths, the computation and operation complexity were evidently reduced, and a simple and custom-built instrument can be easily designed for practical recognition of FHB-damaged wheat kernels. Meanwhile, ASSDN can generate and optimise the high-performance classification network by itself, which is user–friendly and largely expands application potential of deep network.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Wheat kernels damaged by Fusarium head blight (FHB) lose moisture, protein and starch and carry dangerous toxins. Classification of degree of damage can be helpful to customise the use of wheat kernels, reduce financial losses and ensure grain safety. In this study, hyperspectral imaging (HSI) and deep learning network were explored to determine sound, mildly, moderately and severely damaged wheat kernels. Effective wavelengths (EWs) were selected from the reflectance spectroscopy of HSI images by ReliefF, uninformative variable elimination, random frog and shuffled frog leaping algorithm, and the monochromatic images of different combinations of EWs were adopted to develop the classification models combined with an architecture self-search deep network (ASSDN). ASSDN and the images composed of 941, 876 and 732 nm achieved the best determination with average accuracy of 100% and 98.31% in the training and prediction sets, respectively, outperforming other images or methods. And the average area under the curve of 0.9985 indicated its excellent robustness. Using images of sporadic wavelengths, the computation and operation complexity were evidently reduced, and a simple and custom-built instrument can be easily designed for practical recognition of FHB-damaged wheat kernels. Meanwhile, ASSDN can generate and optimise the high-performance classification network by itself, which is user–friendly and largely expands application potential of deep network.
1637.
Zhang, Haoyu; Jin, Yaochu; Jin, Yaochu; Hao, Kuangrong
Evolutionary Search for Complete Neural Network Architectures with Partial Weight Sharing Journal Article
In: IEEE Transactions on Evolutionary Computation, pp. 1-1, 2022.
@article{9672175,
title = {Evolutionary Search for Complete Neural Network Architectures with Partial Weight Sharing},
author = {Haoyu Zhang and Yaochu Jin and Yaochu Jin and Kuangrong Hao},
url = {https://ieeexplore.ieee.org/abstract/document/9672175},
doi = {10.1109/TEVC.2022.3140855},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {IEEE Transactions on Evolutionary Computation},
pages = {1-1},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
1636.
Zhou, Jian; Zheng, Lianyu; Wang, Yiwei; Wang, Cheng; Gao, Robert X.
Automated model generation for machinery fault diagnosis based on reinforcement learning and neural architecture search Journal Article
In: IEEE Transactions on Instrumentation and Measurement, pp. 1-1, 2022.
@article{9673794,
title = {Automated model generation for machinery fault diagnosis based on reinforcement learning and neural architecture search},
author = {Jian Zhou and Lianyu Zheng and Yiwei Wang and Cheng Wang and Robert X. Gao},
url = {https://ieeexplore.ieee.org/abstract/document/9673794},
doi = {10.1109/TIM.2022.3141166},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {IEEE Transactions on Instrumentation and Measurement},
pages = {1-1},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
1635.
Zhan, Cheng; Zhang, Licheng; Zhao, Xin; Lee, Chang-Chun; Huang, Shujiao
Neural Architecture Search for Inversion Technical Report
, 2022.
@techreport{DBLP:journals/corr/abs-2201-01772,
title = {Neural Architecture Search for Inversion},
author = {Cheng Zhan and Licheng Zhang and Xin Zhao and Chang-Chun Lee and Shujiao Huang},
url = {https://arxiv.org/abs/2201.01772},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {CoRR},
volume = {abs/2201.01772},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
1634.
Rorabaugh, Ariel Keller; Caino-Lores, Silvina; Johnston, Travis; Taufer, Michela
Building High-throughput Neural Architecture Search Workflows via a Decoupled Fitness Prediction Engine Journal Article
In: IEEE Transactions on Parallel and Distributed Systems, pp. 1-1, 2022.
@article{9674227,
title = {Building High-throughput Neural Architecture Search Workflows via a Decoupled Fitness Prediction Engine},
author = {Ariel Keller Rorabaugh and Silvina Caino-Lores and Travis Johnston and Michela Taufer},
url = {https://ieeexplore.ieee.org/abstract/document/9674227},
doi = {10.1109/TPDS.2022.3140681},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {IEEE Transactions on Parallel and Distributed Systems},
pages = {1-1},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
1633.
Wang, Xin; Zhang, Ziwei; Zhu, Wenwu
Automated Graph Machine Learning: Approaches, Libraries and Directions Technical Report
, 2022.
@techreport{DBLP:journals/corr/abs-2201-01288,
title = {Automated Graph Machine Learning: Approaches, Libraries and Directions},
author = {Xin Wang and Ziwei Zhang and Wenwu Zhu},
url = {https://arxiv.org/abs/2201.01288},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {CoRR},
volume = {abs/2201.01288},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
1632.
Rorabaugh, Ariel Keller; Caíno-Lores, Silvina; Johnston, Travis; Taufer, Michela
High frequency accuracy and loss data of random neural networks trained on image datasets Journal Article
In: Data in Brief, 40 , pp. 107780, 2022, ISSN: 2352-3409.
@article{RORABAUGH2022107780,
title = {High frequency accuracy and loss data of random neural networks trained on image datasets},
author = {Ariel Keller Rorabaugh and Silvina Caíno-Lores and Travis Johnston and Michela Taufer},
url = {https://www.sciencedirect.com/science/article/pii/S2352340921010544},
doi = {https://doi.org/10.1016/j.dib.2021.107780},
issn = {2352-3409},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {Data in Brief},
volume = {40},
pages = {107780},
abstract = {Neural Networks (NNs) are increasingly used across scientific domains to extract knowledge from experimental or computational data. An NN is composed of natural or artificial neurons that serve as simple processing units and are interconnected into a model architecture; it acquires knowledge from the environment through a learning process and stores this knowledge in its connections. The learning process is conducted by training. During NN training, the learning process can be tracked by periodically validating the NN and calculating its fitness. The resulting sequence of fitness values (i.e., validation accuracy or validation loss) is called the NN learning curve. The development of tools for NN design requires knowledge of diverse NNs and their complete learning curves. Generally, only final fully-trained fitness values for highly accurate NNs are made available to the community, hampering efforts to develop tools for NN design and leaving unaddressed aspects such as explaining the generation of an NN and reproducing its learning process. Our dataset fills this gap by fully recording the structure, metadata, and complete learning curves for a wide variety of random NNs throughout their training. Our dataset captures the lifespan of 6000 NNs throughout generation, training, and validation stages. It consists of a suite of 6000 tables, each table representing the lifespan of one NN. We generate each NN with randomized parameter values and train it for 40 epochs on one of three diverse image datasets (i.e., CIFAR-100, FashionMNIST, SVHN). We calculate and record each NN’s fitness with high frequency—every half epoch—to capture the evolution of the training and validation process. As a result, for each NN, we record the generated parameter values describing the structure of that NN, the image dataset on which the NN trained, and all loss and accuracy values for the NN every half epoch. We put our dataset to the service of researchers studying NN performance and its evolution throughout training and validation. Statistical methods can be applied to our dataset to analyze the shape of learning curves in diverse NNs, and the relationship between an NN’s structure and its fitness. Additionally, the structural data and metadata that we record enable the reconstruction and reproducibility of the associated NN.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Neural Networks (NNs) are increasingly used across scientific domains to extract knowledge from experimental or computational data. An NN is composed of natural or artificial neurons that serve as simple processing units and are interconnected into a model architecture; it acquires knowledge from the environment through a learning process and stores this knowledge in its connections. The learning process is conducted by training. During NN training, the learning process can be tracked by periodically validating the NN and calculating its fitness. The resulting sequence of fitness values (i.e., validation accuracy or validation loss) is called the NN learning curve. The development of tools for NN design requires knowledge of diverse NNs and their complete learning curves. Generally, only final fully-trained fitness values for highly accurate NNs are made available to the community, hampering efforts to develop tools for NN design and leaving unaddressed aspects such as explaining the generation of an NN and reproducing its learning process. Our dataset fills this gap by fully recording the structure, metadata, and complete learning curves for a wide variety of random NNs throughout their training. Our dataset captures the lifespan of 6000 NNs throughout generation, training, and validation stages. It consists of a suite of 6000 tables, each table representing the lifespan of one NN. We generate each NN with randomized parameter values and train it for 40 epochs on one of three diverse image datasets (i.e., CIFAR-100, FashionMNIST, SVHN). We calculate and record each NN’s fitness with high frequency—every half epoch—to capture the evolution of the training and validation process. As a result, for each NN, we record the generated parameter values describing the structure of that NN, the image dataset on which the NN trained, and all loss and accuracy values for the NN every half epoch. We put our dataset to the service of researchers studying NN performance and its evolution throughout training and validation. Statistical methods can be applied to our dataset to analyze the shape of learning curves in diverse NNs, and the relationship between an NN’s structure and its fitness. Additionally, the structural data and metadata that we record enable the reconstruction and reproducibility of the associated NN.
1631.
Zhou, Kaixiong; Liu, Zirui; Duan, Keyu; Hu, Xia
Graph Neural Networks: AutoML Book Chapter
In: pp. 371-389, 2022.
@inbook{Zhou2022,
title = {Graph Neural Networks: AutoML},
author = {Kaixiong Zhou and Zirui Liu and Keyu Duan and Xia Hu},
url = {https://link.springer.com/chapter/10.1007/978-981-16-6054-2_17},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
pages = {371-389},
keywords = {},
pubstate = {published},
tppubtype = {inbook}
}
1630.
Ding, Yadong; Wu, Yu; Huang, Chengyue; Tang, Siliang; Wu, Fei; Yang, Yi; Zhu, Wenwu; Zhuang, Yueting
NAP: Neural Architecture search with Pruning Journal Article
In: Neurocomputing, 2022, ISSN: 0925-2312.
@article{DING2022,
title = {NAP: Neural Architecture search with Pruning},
author = {Yadong Ding and Yu Wu and Chengyue Huang and Siliang Tang and Fei Wu and Yi Yang and Wenwu Zhu and Yueting Zhuang},
url = {https://www.sciencedirect.com/science/article/pii/S0925231221018361},
doi = {https://doi.org/10.1016/j.neucom.2021.12.002},
issn = {0925-2312},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {Neurocomputing},
abstract = {There has been continuously increasing attention attracted by Neural Architecture Search (NAS). Due to its computational efficiency, gradient-based NAS methods like DARTS have become the most popular framework for NAS tasks. Nevertheless, as the search iterates, the derived model in previous NAS frameworks becomes dominated by skip-connects, causing the performance downfall. In this work, we present a novel approach to alleviate this issue, named Neural Architecture search with Pruning (NAP). Unlike prior differentiable architecture search works, our approach draws the idea from network pruning. We first train an over-parameterized network, including all candidate operations. Then we propose a criterion to prune the network. Based on a newly designed relaxation of architecture representation, NAP can derive the most potent model by removing trivial and redundant edges from the whole network topology. Experiments show the effectiveness of our proposed approach. Specifically, the model searched by NAP achieves state-of-the-art performances (2.48% test error) on CIFAR-10. We transfer the model to ImageNet and obtains a 25.1% test error with only 5.0M parameters, which is on par with modern NAS methods.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
There has been continuously increasing attention attracted by Neural Architecture Search (NAS). Due to its computational efficiency, gradient-based NAS methods like DARTS have become the most popular framework for NAS tasks. Nevertheless, as the search iterates, the derived model in previous NAS frameworks becomes dominated by skip-connects, causing the performance downfall. In this work, we present a novel approach to alleviate this issue, named Neural Architecture search with Pruning (NAP). Unlike prior differentiable architecture search works, our approach draws the idea from network pruning. We first train an over-parameterized network, including all candidate operations. Then we propose a criterion to prune the network. Based on a newly designed relaxation of architecture representation, NAP can derive the most potent model by removing trivial and redundant edges from the whole network topology. Experiments show the effectiveness of our proposed approach. Specifically, the model searched by NAP achieves state-of-the-art performances (2.48% test error) on CIFAR-10. We transfer the model to ImageNet and obtains a 25.1% test error with only 5.0M parameters, which is on par with modern NAS methods.
1629.
Reiser, Daniel; Reichel, Peter; Pechmann, Stefan; Mallah, Maen; Oppelt, Maximilian; Hagelauer, Amelie; Breiling, Marco; Fey, Dietmar; Reichenbach, Marc
A Framework for Ultra Low-Power Hardware Accelerators Using NNs for Embedded Time Series Classification Journal Article
In: Journal of Low Power Electronics and Applications, 12 (1), 2022, ISSN: 2079-9268.
@article{jlpea12010002,
title = {A Framework for Ultra Low-Power Hardware Accelerators Using NNs for Embedded Time Series Classification},
author = {Daniel Reiser and Peter Reichel and Stefan Pechmann and Maen Mallah and Maximilian Oppelt and Amelie Hagelauer and Marco Breiling and Dietmar Fey and Marc Reichenbach},
url = {https://www.mdpi.com/2079-9268/12/1/2},
doi = {10.3390/jlpea12010002},
issn = {2079-9268},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {Journal of Low Power Electronics and Applications},
volume = {12},
number = {1},
abstract = {In embedded applications that use neural networks (NNs) for classification tasks, it is important to not only minimize the power consumption of the NN calculation, but of the whole system. Optimization approaches for individual parts exist, such as quantization of the NN or analog calculation of arithmetic operations. However, there is no holistic approach for a complete embedded system design that is generic enough in the design process to be used for different applications, but specific in the hardware implementation to waste no energy for a given application. Therefore, we present a novel framework that allows an end-to-end ASIC implementation of a low-power hardware for time series classification using NNs. This includes a neural architecture search (NAS), which optimizes the NN configuration for accuracy and energy efficiency at the same time. This optimization targets a custom designed hardware architecture that is derived from the key properties of time series classification tasks. Additionally, a hardware generation tool is used that creates a complete system from the definition of the NN. This system uses local multi-level RRAM memory as weight and bias storage to avoid external memory access. Exploiting the non-volatility of these devices, such a system can use a power-down mode to save significant energy during the data acquisition process. Detection of atrial fibrillation (AFib) in electrocardiogram (ECG) data is used as an example for evaluation of the framework. It is shown that a reduction of more than 95% of the energy consumption compared to state-of-the-art solutions is achieved.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
In embedded applications that use neural networks (NNs) for classification tasks, it is important to not only minimize the power consumption of the NN calculation, but of the whole system. Optimization approaches for individual parts exist, such as quantization of the NN or analog calculation of arithmetic operations. However, there is no holistic approach for a complete embedded system design that is generic enough in the design process to be used for different applications, but specific in the hardware implementation to waste no energy for a given application. Therefore, we present a novel framework that allows an end-to-end ASIC implementation of a low-power hardware for time series classification using NNs. This includes a neural architecture search (NAS), which optimizes the NN configuration for accuracy and energy efficiency at the same time. This optimization targets a custom designed hardware architecture that is derived from the key properties of time series classification tasks. Additionally, a hardware generation tool is used that creates a complete system from the definition of the NN. This system uses local multi-level RRAM memory as weight and bias storage to avoid external memory access. Exploiting the non-volatility of these devices, such a system can use a power-down mode to save significant energy during the data acquisition process. Detection of atrial fibrillation (AFib) in electrocardiogram (ECG) data is used as an example for evaluation of the framework. It is shown that a reduction of more than 95% of the energy consumption compared to state-of-the-art solutions is achieved.
1628.
Zhang, Zhen; Liu, Shanghao; Zhang, Yang; Chen, Wenbo
RS-DARTS: A Convolutional Neural Architecture Search for Remote Sensing Image Scene Classification Journal Article
In: Remote Sensing, 14 (1), 2022, ISSN: 2072-4292.
@article{rs14010141,
title = {RS-DARTS: A Convolutional Neural Architecture Search for Remote Sensing Image Scene Classification},
author = {Zhen Zhang and Shanghao Liu and Yang Zhang and Wenbo Chen},
url = {https://www.mdpi.com/2072-4292/14/1/141},
doi = {10.3390/rs14010141},
issn = {2072-4292},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {Remote Sensing},
volume = {14},
number = {1},
abstract = {Due to the superiority of convolutional neural networks, many deep learning methods have been used in image classification. The enormous difference between natural images and remote sensing images makes it difficult to directly utilize or modify existing CNN models for remote sensing scene classification tasks. In this article, a new paradigm is proposed that can automatically design a suitable CNN architecture for scene classification. A more efficient search framework, RS-DARTS, is adopted to find the optimal network architecture. This framework has two phases. In the search phase, some new strategies are presented, making the calculation process smoother, and better distinguishing the optimal and other operations. In addition, we added noise to suppress skip connections in order to close the gap between trained and validation processing and ensure classification accuracy. Moreover, a small part of the neural network is sampled to reduce the redundancy in exploring the network space and speed up the search processing. In the evaluation phase, the optimal cell architecture is stacked to construct the final network. Extensive experiments demonstrated the validity of the search strategy and the impressive classification performance of RS-DARTS on four public benchmark datasets. The proposed method showed more effectiveness than the manually designed CNN model and other methods of neural architecture search. Especially, in terms of search cost, RS-DARTS consumed less time than other NAS methods.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Due to the superiority of convolutional neural networks, many deep learning methods have been used in image classification. The enormous difference between natural images and remote sensing images makes it difficult to directly utilize or modify existing CNN models for remote sensing scene classification tasks. In this article, a new paradigm is proposed that can automatically design a suitable CNN architecture for scene classification. A more efficient search framework, RS-DARTS, is adopted to find the optimal network architecture. This framework has two phases. In the search phase, some new strategies are presented, making the calculation process smoother, and better distinguishing the optimal and other operations. In addition, we added noise to suppress skip connections in order to close the gap between trained and validation processing and ensure classification accuracy. Moreover, a small part of the neural network is sampled to reduce the redundancy in exploring the network space and speed up the search processing. In the evaluation phase, the optimal cell architecture is stacked to construct the final network. Extensive experiments demonstrated the validity of the search strategy and the impressive classification performance of RS-DARTS on four public benchmark datasets. The proposed method showed more effectiveness than the manually designed CNN model and other methods of neural architecture search. Especially, in terms of search cost, RS-DARTS consumed less time than other NAS methods.
1627.
Chen, Dong; Shen, Hao; Shen, Yuchen
PT-NAS: Designing efficient keypoint-based object detectors for desktop CPU platforms Journal Article
In: Neurocomputing, 476 , pp. 38-52, 2022, ISSN: 0925-2312.
@article{CHEN202238,
title = {PT-NAS: Designing efficient keypoint-based object detectors for desktop CPU platforms},
author = {Dong Chen and Hao Shen and Yuchen Shen},
url = {https://www.sciencedirect.com/science/article/pii/S0925231221019238},
doi = {https://doi.org/10.1016/j.neucom.2021.12.067},
issn = {0925-2312},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {Neurocomputing},
volume = {476},
pages = {38-52},
abstract = {Recently, keypoint-based object detectors have attracted widespread attention, due to their novel structure and excellent performance. However, in terms of their design, there are still two limitations: 1) Most keypoint-based methods are designed for GPU platforms, which makes them inefficient on desktop CPU platforms. 2) Existing works still rely heavily on manual design and prior knowledge. To this end, this work aims to offer a practical solution for designing CPU-efficient key-point detectors. First, we present a set of practical design guidelines by comparing different detection architectures. Following the proposed guidelines, we further develop a progressive three-phase network architecture search (PT-NAS) to achieve the automated design of detection architectures. Benefiting from our hierarchical search space and novel search pipeline, our PT-NAS not only achieves higher search efficiency, but also satisfies the practicality of CPU platforms. On the MS-COCO benchmark, we utilize our PT-NAS to generate several key-point detectors for fast inference on desktop CPUs. Finally, comprehensive comparison experiments prove that the proposed PT-NAS can produce new state-of-the-art keypoint-based detectors for CPU platforms.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Recently, keypoint-based object detectors have attracted widespread attention, due to their novel structure and excellent performance. However, in terms of their design, there are still two limitations: 1) Most keypoint-based methods are designed for GPU platforms, which makes them inefficient on desktop CPU platforms. 2) Existing works still rely heavily on manual design and prior knowledge. To this end, this work aims to offer a practical solution for designing CPU-efficient key-point detectors. First, we present a set of practical design guidelines by comparing different detection architectures. Following the proposed guidelines, we further develop a progressive three-phase network architecture search (PT-NAS) to achieve the automated design of detection architectures. Benefiting from our hierarchical search space and novel search pipeline, our PT-NAS not only achieves higher search efficiency, but also satisfies the practicality of CPU platforms. On the MS-COCO benchmark, we utilize our PT-NAS to generate several key-point detectors for fast inference on desktop CPUs. Finally, comprehensive comparison experiments prove that the proposed PT-NAS can produce new state-of-the-art keypoint-based detectors for CPU platforms.
1626.
Zhang, Yang; Zong, Ruohan; Kou, Ziyi; Shang, Lanyu; Wang, Dong
On streaming disaster damage assessment in social sensing: A crowd-driven dynamic neural architecture searching approach Journal Article
In: Knowledge-Based Systems, 239 , pp. 107984, 2022, ISSN: 0950-7051.
@article{ZHANG2022107984,
title = {On streaming disaster damage assessment in social sensing: A crowd-driven dynamic neural architecture searching approach},
author = {Yang Zhang and Ruohan Zong and Ziyi Kou and Lanyu Shang and Dong Wang},
url = {https://www.sciencedirect.com/science/article/pii/S0950705121011023},
doi = {https://doi.org/10.1016/j.knosys.2021.107984},
issn = {0950-7051},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {Knowledge-Based Systems},
volume = {239},
pages = {107984},
abstract = {Motivated by the recent advances in Internet and communication techniques and the proliferation of online social media, social sensing has emerged as a new sensing paradigm to obtain timely observations of the physical world from “human sensors”. In this study, we focus on an emerging application in social sensing – streaming disaster damage assessment (DDA), which aims to automatically assess the damage severity of affected areas in a disaster event on the fly by leveraging the streaming imagery data about the disaster on social media. In particular, we study a dynamic optimal neural architecture searching (NAS) problem in streaming DDA applications. Our goal is to dynamically determine the optimal neural network architecture that accurately estimates the damage severity for each newly arrived image in the stream by leveraging human intelligence from the crowdsourcing systems. The present study is motivated by the observation that the neural network architectures in current DDA solutions are mainly designed by artificial intelligence (AI) experts, which often leads to non-negligible costs and errors given the dynamic nature of the streaming DDA applications and the lack of real-time annotations of the massive social media data inputs. Two critical technical challenges exist in solving our problem: (i) it is non-trivial to dynamically identify the optimal neural network architecture for each image on the fly without knowing its ground-truth label a priori; (ii) it is challenging to effectively leverage the imperfect crowd intelligence to correctly identify the optimal neural network architecture for each image. To address the above challenges, we developed CD-NAS, a dynamic crowd-AI collaborative NAS framework that carefully explores the human intelligence from crowdsourcing systems to solve the dynamic optimal NAS problem and optimize the performance of streaming DDA applications. The evaluation results from a real-world streaming DDA application show that CD-NAS consistently outperforms the state-of-the-art AI and NAS baselines by achieving the highest disaster damage assessment accuracy while maintaining the lowest computational cost.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Motivated by the recent advances in Internet and communication techniques and the proliferation of online social media, social sensing has emerged as a new sensing paradigm to obtain timely observations of the physical world from “human sensors”. In this study, we focus on an emerging application in social sensing – streaming disaster damage assessment (DDA), which aims to automatically assess the damage severity of affected areas in a disaster event on the fly by leveraging the streaming imagery data about the disaster on social media. In particular, we study a dynamic optimal neural architecture searching (NAS) problem in streaming DDA applications. Our goal is to dynamically determine the optimal neural network architecture that accurately estimates the damage severity for each newly arrived image in the stream by leveraging human intelligence from the crowdsourcing systems. The present study is motivated by the observation that the neural network architectures in current DDA solutions are mainly designed by artificial intelligence (AI) experts, which often leads to non-negligible costs and errors given the dynamic nature of the streaming DDA applications and the lack of real-time annotations of the massive social media data inputs. Two critical technical challenges exist in solving our problem: (i) it is non-trivial to dynamically identify the optimal neural network architecture for each image on the fly without knowing its ground-truth label a priori; (ii) it is challenging to effectively leverage the imperfect crowd intelligence to correctly identify the optimal neural network architecture for each image. To address the above challenges, we developed CD-NAS, a dynamic crowd-AI collaborative NAS framework that carefully explores the human intelligence from crowdsourcing systems to solve the dynamic optimal NAS problem and optimize the performance of streaming DDA applications. The evaluation results from a real-world streaming DDA application show that CD-NAS consistently outperforms the state-of-the-art AI and NAS baselines by achieving the highest disaster damage assessment accuracy while maintaining the lowest computational cost.
1625.
Sivangi, Kaushik Bhargav; Dasari, Chandra Mohan; Amilpur, Santhosh; Bhukya, Raju
NoAS-DS: Neural optimal architecture search for detection of diverse DNA signals Journal Article
In: Neural Networks, 147 , pp. 63-71, 2022, ISSN: 0893-6080.
@article{SIVANGI202263,
title = {NoAS-DS: Neural optimal architecture search for detection of diverse DNA signals},
author = {Kaushik Bhargav Sivangi and Chandra Mohan Dasari and Santhosh Amilpur and Raju Bhukya},
url = {https://www.sciencedirect.com/science/article/pii/S0893608021004822},
doi = {https://doi.org/10.1016/j.neunet.2021.12.009},
issn = {0893-6080},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {Neural Networks},
volume = {147},
pages = {63-71},
abstract = {Neural network architectures are high-performing variable models that can solve many learning tasks. Designing architectures manually require substantial time and also prior knowledge and expertise to develop a high-accuracy model. Most of the architecture search methods are developed over the task of image classification resulting in the building of complex architectures intended for large data inputs such as images. Motivated by the applications of DNA computing in Neural Architecture Search (NAS), we propose NoAS-DS which is specifically built for the architecture search of sequence-based classification tasks. Furthermore, NoAS-DS is applied to the task of predicting binding sites. Unlike other methods that implement only Convolution layers, NoAS-DS, specifically combines Convolution and LSTM layers that helps in the process of automatic architecture building. This hybrid approach helped in achieving high accuracy results on TFBS and RBP datasets which outperformed other models in TF-DNA binding prediction tasks. The best architectures generated by the proposed model can be applied to other DNA datasets of similar nature using transfer learning technique that demonstrates its generalization capability. This greatly reduces the effort required to build new architectures for other prediction tasks.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Neural network architectures are high-performing variable models that can solve many learning tasks. Designing architectures manually require substantial time and also prior knowledge and expertise to develop a high-accuracy model. Most of the architecture search methods are developed over the task of image classification resulting in the building of complex architectures intended for large data inputs such as images. Motivated by the applications of DNA computing in Neural Architecture Search (NAS), we propose NoAS-DS which is specifically built for the architecture search of sequence-based classification tasks. Furthermore, NoAS-DS is applied to the task of predicting binding sites. Unlike other methods that implement only Convolution layers, NoAS-DS, specifically combines Convolution and LSTM layers that helps in the process of automatic architecture building. This hybrid approach helped in achieving high accuracy results on TFBS and RBP datasets which outperformed other models in TF-DNA binding prediction tasks. The best architectures generated by the proposed model can be applied to other DNA datasets of similar nature using transfer learning technique that demonstrates its generalization capability. This greatly reduces the effort required to build new architectures for other prediction tasks.
1624.
Liu, Shengfeng; Ge, Fangfei; Zhao, Lin; Wang, Tianfu; Ni, Dong; Liu, Tianming
NAS-optimized topology-preserving transfer learning for differentiating cortical folding patterns Journal Article
In: Medical Image Analysis, 77 , pp. 102316, 2022, ISSN: 1361-8415.
@article{LIU2022102316,
title = {NAS-optimized topology-preserving transfer learning for differentiating cortical folding patterns},
author = {Shengfeng Liu and Fangfei Ge and Lin Zhao and Tianfu Wang and Dong Ni and Tianming Liu},
url = {https://www.sciencedirect.com/science/article/pii/S1361841521003613},
doi = {https://doi.org/10.1016/j.media.2021.102316},
issn = {1361-8415},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {Medical Image Analysis},
volume = {77},
pages = {102316},
abstract = {Increasing evidence suggests that cortical folding patterns of human cerebral cortex manifest overt structural and functional differences. However, for interpretability, few studies leverage advanced techniques (e.g., deep learning) to investigate the difference among cortical folds, resulting in more differences yet to be extensively explored. To this end, we proposed an effective topology-preserving transfer learning framework to differentiate cortical fMRI time series extracted from cortical folds. Our framework consists of three main parts: (1) Neural architecture search (NAS), which is used to devise a well-performing network structure based on an initialized hand-designed super-graph in an image dataset; (2) Topology-preserving transfer, which takes the model searched by NAS as the source network, keeping the topological connectivity in the network unchanged, while transforming all 2D operations including convolution and pooling into 1D, therefore resulting in a topology-preserving network, named TPNAS-Net; (3) Classification and correlation analysis, which involves using the TPNAS-Net to classify 1D cortical fMRI time series for each individual brain, and performing a group difference analysis between autism spectrum disorder (ASD) and healthy control (HC) and correlation analysis with clinical information (i.e., age). Extensive experiments on two ASD datasets obtain consistent results, demonstrating that the TPNAS-Net not only discriminates cortical folding patterns at high classification accuracy, but also captures subtle differences between ASD and HC (p-value = 0.042). In addition, we discover that there is a positive correlation between the classification accuracy and age in ASD (r = 0.39},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Increasing evidence suggests that cortical folding patterns of human cerebral cortex manifest overt structural and functional differences. However, for interpretability, few studies leverage advanced techniques (e.g., deep learning) to investigate the difference among cortical folds, resulting in more differences yet to be extensively explored. To this end, we proposed an effective topology-preserving transfer learning framework to differentiate cortical fMRI time series extracted from cortical folds. Our framework consists of three main parts: (1) Neural architecture search (NAS), which is used to devise a well-performing network structure based on an initialized hand-designed super-graph in an image dataset; (2) Topology-preserving transfer, which takes the model searched by NAS as the source network, keeping the topological connectivity in the network unchanged, while transforming all 2D operations including convolution and pooling into 1D, therefore resulting in a topology-preserving network, named TPNAS-Net; (3) Classification and correlation analysis, which involves using the TPNAS-Net to classify 1D cortical fMRI time series for each individual brain, and performing a group difference analysis between autism spectrum disorder (ASD) and healthy control (HC) and correlation analysis with clinical information (i.e., age). Extensive experiments on two ASD datasets obtain consistent results, demonstrating that the TPNAS-Net not only discriminates cortical folding patterns at high classification accuracy, but also captures subtle differences between ASD and HC (p-value = 0.042). In addition, we discover that there is a positive correlation between the classification accuracy and age in ASD (r = 0.39
1623.
Fan, Zhenkun; Hu, Guosheng; Sun, Xin; Wang, Gaige; Dong, Junyu; Su, Chi
Self-attention neural architecture search for semantic image segmentation Journal Article
In: Knowledge-Based Systems, 239 , pp. 107968, 2022, ISSN: 0950-7051.
@article{FAN2022107968,
title = {Self-attention neural architecture search for semantic image segmentation},
author = {Zhenkun Fan and Guosheng Hu and Xin Sun and Gaige Wang and Junyu Dong and Chi Su},
url = {https://www.sciencedirect.com/science/article/pii/S0950705121010947},
doi = {https://doi.org/10.1016/j.knosys.2021.107968},
issn = {0950-7051},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {Knowledge-Based Systems},
volume = {239},
pages = {107968},
abstract = {Self-attention can capture long-distance dependencies and is widely used in semantic segmentation. Existing methods mainly use two kinds of self-attentions, i.e., spatial attention and channel attention, which can capture the relations in HW dimension (image plane, height and width) and C dimension (channels), respectively. Very little research investigates self-attention along other dimensions, which can potentially improve the segmentation performance. In this work, we investigate the self-attentions along all the possible dimensions H,W,C,HW,HC,CW,HWC. Then we explore the aggregation of all the possible self-attentions. We apply the neural architecture search (NAS) technique to achieve optimal aggregation. Specifically, we carefully design (1) the search space and (2) the optimization method. For (1), we introduce a building block, a basic self-attention search unit (BSU), which can model self-attentions along all the dimensions. And the search space contains within-BSU and cross-BSU operations. In addition, we propose an attention-map splitting method, which can reduce the computations by 1/3. For (2), we apply an efficient differentiable optimization method to search the optimal aggregation. We conduct extensive experiments on Cityscapes and ADE20K datasets. The results show the effectiveness of the proposed method, and we achieve very competitive performance against state-of-the-art methods.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Self-attention can capture long-distance dependencies and is widely used in semantic segmentation. Existing methods mainly use two kinds of self-attentions, i.e., spatial attention and channel attention, which can capture the relations in HW dimension (image plane, height and width) and C dimension (channels), respectively. Very little research investigates self-attention along other dimensions, which can potentially improve the segmentation performance. In this work, we investigate the self-attentions along all the possible dimensions H,W,C,HW,HC,CW,HWC. Then we explore the aggregation of all the possible self-attentions. We apply the neural architecture search (NAS) technique to achieve optimal aggregation. Specifically, we carefully design (1) the search space and (2) the optimization method. For (1), we introduce a building block, a basic self-attention search unit (BSU), which can model self-attentions along all the dimensions. And the search space contains within-BSU and cross-BSU operations. In addition, we propose an attention-map splitting method, which can reduce the computations by 1/3. For (2), we apply an efficient differentiable optimization method to search the optimal aggregation. We conduct extensive experiments on Cityscapes and ADE20K datasets. The results show the effectiveness of the proposed method, and we achieve very competitive performance against state-of-the-art methods.
1622.
Baymurzina, Dilyara; Golikov, Eugene; Burtsev, Mikhail
A review of neural architecture search Journal Article
In: Neurocomputing, 474 , pp. 82-93, 2022, ISSN: 0925-2312.
@article{BAYMURZINA202282,
title = {A review of neural architecture search},
author = {Dilyara Baymurzina and Eugene Golikov and Mikhail Burtsev},
url = {https://www.sciencedirect.com/science/article/pii/S0925231221018439},
doi = {https://doi.org/10.1016/j.neucom.2021.12.014},
issn = {0925-2312},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {Neurocomputing},
volume = {474},
pages = {82-93},
abstract = {Despite the impressive progress in neural network architecture design, improving the performance of the existing state-of-the-art models has become increasingly challenging. For this reason, the paradigm for neural architecture design is shifting from being expert-driven to almost fully automated. An emerging body of research related to such machine-aided design is called a Neural Architecture Search (NAS). This paper reviews the recent works on NAS and highlights several crucial concepts and problems of this field.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Despite the impressive progress in neural network architecture design, improving the performance of the existing state-of-the-art models has become increasingly challenging. For this reason, the paradigm for neural architecture design is shifting from being expert-driven to almost fully automated. An emerging body of research related to such machine-aided design is called a Neural Architecture Search (NAS). This paper reviews the recent works on NAS and highlights several crucial concepts and problems of this field.
1621.
Oloulade, Babatounde Moctard; Gao, Jianliang; Chen, Jiamin; Lyu, Tengfei; Al-Sabri, Raeed
Graph neural architecture search: A survey Journal Article
In: Tsinghua Science and Technology, 27 (4), pp. 692-708, 2022.
@article{9645440,
title = {Graph neural architecture search: A survey},
author = {Babatounde Moctard Oloulade and Jianliang Gao and Jiamin Chen and Tengfei Lyu and Raeed Al-Sabri},
url = {https://ieeexplore.ieee.org/document/9645440},
doi = {10.26599/TST.2021.9010057},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {Tsinghua Science and Technology},
volume = {27},
number = {4},
pages = {692-708},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
1620.
Xue, Chao; Hu, Mengting; Huang, Xueqi; Li, Chun-Guang
Automated search space and search strategy selection for AutoML Journal Article
In: Pattern Recognition, 124 , pp. 108474, 2022, ISSN: 0031-3203.
@article{XUE2022108474,
title = {Automated search space and search strategy selection for AutoML},
author = {Chao Xue and Mengting Hu and Xueqi Huang and Chun-Guang Li},
url = {https://www.sciencedirect.com/science/article/pii/S0031320321006506},
doi = {https://doi.org/10.1016/j.patcog.2021.108474},
issn = {0031-3203},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {Pattern Recognition},
volume = {124},
pages = {108474},
abstract = {Existing works on Automated Machine Learning (AutoML) are mainly based on predefined search space. This paper seeks synergetic automation of two ingredients, i.e., search space and search strategies. Specifically, we formulate the automation of search space and search strategies as a combinatorial optimization problem. Our empirical study on many architecture benchmarks shows that identifying the suitable search space exerts more effect than choosing a sophisticated search strategy. Motivated by this, we attempt to leverage a machine learning method to solve the discrete optimization problem, and thus develop a Layered Architecture Search Tree (LArST) approach to synergize these two components. In addition, we use a probe model-based method to extract dataset-wise features, i.e., meta-features, which is able to facilitate the estimation of proper search space and search strategy for a given task. Experimental results show the efficacy of our approach under different search mechanisms and various datasets and hardware platforms.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Existing works on Automated Machine Learning (AutoML) are mainly based on predefined search space. This paper seeks synergetic automation of two ingredients, i.e., search space and search strategies. Specifically, we formulate the automation of search space and search strategies as a combinatorial optimization problem. Our empirical study on many architecture benchmarks shows that identifying the suitable search space exerts more effect than choosing a sophisticated search strategy. Motivated by this, we attempt to leverage a machine learning method to solve the discrete optimization problem, and thus develop a Layered Architecture Search Tree (LArST) approach to synergize these two components. In addition, we use a probe model-based method to extract dataset-wise features, i.e., meta-features, which is able to facilitate the estimation of proper search space and search strategy for a given task. Experimental results show the efficacy of our approach under different search mechanisms and various datasets and hardware platforms.
1619.
Li, Xudong; Zheng, Jianhua; Li, Mingtao; Ma, Wenzhen; Hu, Yang
One-shot neural architecture search for fault diagnosis using vibration signals Journal Article
In: Expert Systems with Applications, 190 , pp. 116027, 2022, ISSN: 0957-4174.
@article{LI2022116027,
title = {One-shot neural architecture search for fault diagnosis using vibration signals},
author = {Xudong Li and Jianhua Zheng and Mingtao Li and Wenzhen Ma and Yang Hu},
url = {https://www.sciencedirect.com/science/article/pii/S0957417421013737},
doi = {https://doi.org/10.1016/j.eswa.2021.116027},
issn = {0957-4174},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {Expert Systems with Applications},
volume = {190},
pages = {116027},
abstract = {Machine learning method has been widely applied in industrial fault diagnosis, especially the deep learning method. In the field of industrial fault diagnosis, deep learning is mostly used to extract features of vibration signals to achieve end-to-end fault diagnosis systems. Due to the complexity and variety of actual industrial datasets, some deep learning models are designed to be complicated. However, designing neural network architectures requires rich professional knowledge, experience, and a large number of experiments, increasing the difficulty of developing deep learning models. Fortunately, Neural Architecture Search (NAS), a branch of Automated Machine Learning (AutoML), is developing rapidly. Given a search space, NAS can search for networks that perform better than manually designed. In this paper, a one-shot NAS method for fault diagnosis is proposed. The one-shot model is a supernet that contains all candidate networks in a given search space. The supernet is trained to evaluate the actual performance of candidate networks by measuring the difference between its output probability and the true labels. According to the prediction of supernet, the networks with excellent performance can be searched, using some common search methods such as random search or evolutionary algorithm. Finally, the searched network is trained by reusing the weights of the supernet. To evaluate the proposed method, two search spaces are designed, ResNet and Inception search spaces, to search on PHM 2009 Data Challenge gearbox dataset. The state-of-the-art results are obtained, and accuracies of searched ResNet-A and Inception-A are 84.11% and 83.81%, which are 3.29% and 10.88% higher than Reinforcement Learning based NAS.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Machine learning method has been widely applied in industrial fault diagnosis, especially the deep learning method. In the field of industrial fault diagnosis, deep learning is mostly used to extract features of vibration signals to achieve end-to-end fault diagnosis systems. Due to the complexity and variety of actual industrial datasets, some deep learning models are designed to be complicated. However, designing neural network architectures requires rich professional knowledge, experience, and a large number of experiments, increasing the difficulty of developing deep learning models. Fortunately, Neural Architecture Search (NAS), a branch of Automated Machine Learning (AutoML), is developing rapidly. Given a search space, NAS can search for networks that perform better than manually designed. In this paper, a one-shot NAS method for fault diagnosis is proposed. The one-shot model is a supernet that contains all candidate networks in a given search space. The supernet is trained to evaluate the actual performance of candidate networks by measuring the difference between its output probability and the true labels. According to the prediction of supernet, the networks with excellent performance can be searched, using some common search methods such as random search or evolutionary algorithm. Finally, the searched network is trained by reusing the weights of the supernet. To evaluate the proposed method, two search spaces are designed, ResNet and Inception search spaces, to search on PHM 2009 Data Challenge gearbox dataset. The state-of-the-art results are obtained, and accuracies of searched ResNet-A and Inception-A are 84.11% and 83.81%, which are 3.29% and 10.88% higher than Reinforcement Learning based NAS.
1618.
Grahn, Lea; Rachner, Jonas; Göppert, Amon; Saeed, Sazvan; Schmitt, Robert H.
Framework for Potential Analysis by Approximating Line-Less Assembly Systems with AutoML Inproceedings
In: Ändersen, Ann-Louise; Andersen, Rasmus; Brunoe, Thomas Ditlev; Larsen, Maria Stoettrup Schioenning; Nielsen, Kjeld; Napoleone, Alessia; Kjeldgaard, Stefan" (Ed.): Towards Sustainable Customization: Bridging Smart Products and Manufacturing Systems, pp. 423–430, Springer International Publishing, Cham, 2022, ISBN: 978-3-030-90700-6.
@inproceedings{10.1007/978-3-030-90700-6_48,
title = {Framework for Potential Analysis by Approximating Line-Less Assembly Systems with AutoML},
author = {Lea Grahn and Jonas Rachner and Amon Göppert and Sazvan Saeed and Robert H. Schmitt},
editor = {Ann-Louise Ändersen and Rasmus Andersen and Thomas Ditlev Brunoe and Maria Stoettrup Schioenning Larsen and Kjeld Nielsen and Alessia Napoleone and Stefan" Kjeldgaard},
url = {https://link.springer.com/chapter/10.1007/978-3-030-90700-6_48},
isbn = {978-3-030-90700-6},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
booktitle = {Towards Sustainable Customization: Bridging Smart Products and Manufacturing Systems},
pages = {423--430},
publisher = {Springer International Publishing},
address = {Cham},
abstract = {Ässembly systems are required to be more flexible due to increasing product variety. This is accomplished by breaking up the prevailing rigid linking of assembly stations in classic line configurations to line-less assembly systems (LAS). For a quantified potential analysis, it is necessary to assess the dependency of performance indicators on input parameters in a large number of production scenarios. Existing methods are either experience-based or computationally expensive due to full-factorial experiment plans. Therefore, the contribution of this paper is threefold. First, a seamlessly automated scenario analysis tool is developed, simulating a large set of assembly scenarios using a discrete-event simulation. Second, an artificial neural network (ANN) based approximation pipeline of the scenario analysis is implemented. An integrated AutoML pipeline for hyperparameter optimization (HPO) and Neural Architecture Search (NAS) allows for a faster potential approximation with sufficiently accurate prediction. Last, an integrated decision support system including the mentioned components is defined. It includes a priori planning of the overall system and allows the assessment of adaptions in reaction to the current system status."},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Ässembly systems are required to be more flexible due to increasing product variety. This is accomplished by breaking up the prevailing rigid linking of assembly stations in classic line configurations to line-less assembly systems (LAS). For a quantified potential analysis, it is necessary to assess the dependency of performance indicators on input parameters in a large number of production scenarios. Existing methods are either experience-based or computationally expensive due to full-factorial experiment plans. Therefore, the contribution of this paper is threefold. First, a seamlessly automated scenario analysis tool is developed, simulating a large set of assembly scenarios using a discrete-event simulation. Second, an artificial neural network (ANN) based approximation pipeline of the scenario analysis is implemented. An integrated AutoML pipeline for hyperparameter optimization (HPO) and Neural Architecture Search (NAS) allows for a faster potential approximation with sufficiently accurate prediction. Last, an integrated decision support system including the mentioned components is defined. It includes a priori planning of the overall system and allows the assessment of adaptions in reaction to the current system status."
1617.
Cheng, Guoan; Matsune, Ai; Du, Hao; Liu, XinZhi; Zhan, Shu
Exploring more diverse network architectures for single image super-resolution Journal Article
In: Knowledge-Based Systems, 235 , pp. 107648, 2022, ISSN: 0950-7051.
@article{CHENG2022107648,
title = {Exploring more diverse network architectures for single image super-resolution},
author = {Guoan Cheng and Ai Matsune and Hao Du and XinZhi Liu and Shu Zhan},
url = {https://www.sciencedirect.com/science/article/pii/S0950705121009102},
doi = {https://doi.org/10.1016/j.knosys.2021.107648},
issn = {0950-7051},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {Knowledge-Based Systems},
volume = {235},
pages = {107648},
abstract = {We propose a plug-and-play neural architecture search (NAS) method to explore diverse architectures for single image super-resolution (SISR). Unlike current NAS-based methods with the single path setting and pipeline setting, our proposed method achieves the trade-off between diverse network architectures and search cost. Our proposed method formulates the task in a differentiable manner, which inherits the architecture parameter optimization method from Discrete Stochastic Neural Architecture Search (DSNAS). Besides the straightforward searching of operations, we also search each node in a cell for the activation function, from-node, and skip-connection node, which diverse the searched architecture topologies. The individually searching of skip-connection node avoids skip-connection excessive phenomenon. Moreover, to alleviate the influence of inconsistent architecture between training and testing periods, we introduce random variables into the architecture parameter as regularization. Benchmark experiments show our state-of-the-art performance under specific parameters and FLOPs constraints. Compared with other NAS-based SISR methods, our proposed methods achieve better performance with less searching time and resources. The superior results further demonstrate the effectiveness of our proposed NAS methods.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
We propose a plug-and-play neural architecture search (NAS) method to explore diverse architectures for single image super-resolution (SISR). Unlike current NAS-based methods with the single path setting and pipeline setting, our proposed method achieves the trade-off between diverse network architectures and search cost. Our proposed method formulates the task in a differentiable manner, which inherits the architecture parameter optimization method from Discrete Stochastic Neural Architecture Search (DSNAS). Besides the straightforward searching of operations, we also search each node in a cell for the activation function, from-node, and skip-connection node, which diverse the searched architecture topologies. The individually searching of skip-connection node avoids skip-connection excessive phenomenon. Moreover, to alleviate the influence of inconsistent architecture between training and testing periods, we introduce random variables into the architecture parameter as regularization. Benchmark experiments show our state-of-the-art performance under specific parameters and FLOPs constraints. Compared with other NAS-based SISR methods, our proposed methods achieve better performance with less searching time and resources. The superior results further demonstrate the effectiveness of our proposed NAS methods.
1616.
Chen, Haoze; Zhang, Zhijie; Jin, Mingwu; Wang, Fengxiang
Prediction of dMRI signals with neural architecture search Journal Article
In: Journal of Neuroscience Methods, 365 , pp. 109389, 2022, ISSN: 0165-0270.
@article{CHEN2022109389,
title = {Prediction of dMRI signals with neural architecture search},
author = {Haoze Chen and Zhijie Zhang and Mingwu Jin and Fengxiang Wang},
url = {https://www.sciencedirect.com/science/article/pii/S0165027021003241},
doi = {https://doi.org/10.1016/j.jneumeth.2021.109389},
issn = {0165-0270},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {Journal of Neuroscience Methods},
volume = {365},
pages = {109389},
abstract = {Background
There is growing interest in the neuroscience community in estimating and mapping microscopic properties of brain tissue non-invasively using magnetic resonance measurements. Machine learning methods are actively investigated to predict the signals measured in diffusion magnetic resonance imaging (dMRI).
New method
We applied the neural architecture search (NAS) to train a recurrent neural network to generate a multilayer perceptron to predict the dMRI data of unknown signals based on the different acquisition parameters and training data. The search space of NAS is the number of neurons in each layer of the multilayer perceptron network. To our best knowledge, this is the first time to apply NAS to solve the dMRI signal prediction problem.
Results
The experimental results demonstrate that the proposed NAS method can achieve fast training and predict dMRI signals accurately. For dMRI signals with four acquisition strategies of double diffusion encoding (DDE), double oscillating diffusion encoding (DODE), multi-shell and DSI-like pulsed gradient spin-echo (PGSE), the mean squared errors of the multilayer perceptron network designed by NAS are 0.0043, 0.0034, 0.0147 and 0.0199, respectively.
Comparison with existing method(s)
We also compared NAS with other machine learning prediction methods, such as support vector regression (SVR), decision tree (DT) and random forest (RF), k-nearest neighbors (KNN), adaboost regressor (AR), gradient boosting regressor (GBR) and extra-trees regressor (ET). NAS achieved the better prediction performance in most cases.
Conclusion
In this study, NAS was developed for the prediction of dMRI signals and could become an effective prediction tool.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Background
There is growing interest in the neuroscience community in estimating and mapping microscopic properties of brain tissue non-invasively using magnetic resonance measurements. Machine learning methods are actively investigated to predict the signals measured in diffusion magnetic resonance imaging (dMRI).
New method
We applied the neural architecture search (NAS) to train a recurrent neural network to generate a multilayer perceptron to predict the dMRI data of unknown signals based on the different acquisition parameters and training data. The search space of NAS is the number of neurons in each layer of the multilayer perceptron network. To our best knowledge, this is the first time to apply NAS to solve the dMRI signal prediction problem.
Results
The experimental results demonstrate that the proposed NAS method can achieve fast training and predict dMRI signals accurately. For dMRI signals with four acquisition strategies of double diffusion encoding (DDE), double oscillating diffusion encoding (DODE), multi-shell and DSI-like pulsed gradient spin-echo (PGSE), the mean squared errors of the multilayer perceptron network designed by NAS are 0.0043, 0.0034, 0.0147 and 0.0199, respectively.
Comparison with existing method(s)
We also compared NAS with other machine learning prediction methods, such as support vector regression (SVR), decision tree (DT) and random forest (RF), k-nearest neighbors (KNN), adaboost regressor (AR), gradient boosting regressor (GBR) and extra-trees regressor (ET). NAS achieved the better prediction performance in most cases.
Conclusion
In this study, NAS was developed for the prediction of dMRI signals and could become an effective prediction tool.
There is growing interest in the neuroscience community in estimating and mapping microscopic properties of brain tissue non-invasively using magnetic resonance measurements. Machine learning methods are actively investigated to predict the signals measured in diffusion magnetic resonance imaging (dMRI).
New method
We applied the neural architecture search (NAS) to train a recurrent neural network to generate a multilayer perceptron to predict the dMRI data of unknown signals based on the different acquisition parameters and training data. The search space of NAS is the number of neurons in each layer of the multilayer perceptron network. To our best knowledge, this is the first time to apply NAS to solve the dMRI signal prediction problem.
Results
The experimental results demonstrate that the proposed NAS method can achieve fast training and predict dMRI signals accurately. For dMRI signals with four acquisition strategies of double diffusion encoding (DDE), double oscillating diffusion encoding (DODE), multi-shell and DSI-like pulsed gradient spin-echo (PGSE), the mean squared errors of the multilayer perceptron network designed by NAS are 0.0043, 0.0034, 0.0147 and 0.0199, respectively.
Comparison with existing method(s)
We also compared NAS with other machine learning prediction methods, such as support vector regression (SVR), decision tree (DT) and random forest (RF), k-nearest neighbors (KNN), adaboost regressor (AR), gradient boosting regressor (GBR) and extra-trees regressor (ET). NAS achieved the better prediction performance in most cases.
Conclusion
In this study, NAS was developed for the prediction of dMRI signals and could become an effective prediction tool.
1615.
Nistor, Sergiu Cosmin; Czibula, Gabriela
IntelliSwAS: Optimizing deep neural network architectures using a particle swarm-based approach Journal Article
In: Expert Systems with Applications, 187 , pp. 115945, 2022, ISSN: 0957-4174.
@article{NISTOR2022115945,
title = {IntelliSwAS: Optimizing deep neural network architectures using a particle swarm-based approach},
author = {Sergiu Cosmin Nistor and Gabriela Czibula},
url = {https://www.sciencedirect.com/science/article/pii/S0957417421012987},
doi = {https://doi.org/10.1016/j.eswa.2021.115945},
issn = {0957-4174},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {Expert Systems with Applications},
volume = {187},
pages = {115945},
abstract = {Deep learning is in a continuous evolution and many domains benefit from this substantial progress in the development of intelligent solutions. While this progress has been swift, there are more and more applications and the specific requirements of each application domain entails much work done by researchers to adapt existing models or create new ones. This traditional approach has produced many successful designs, but recently, automated methods of finding neural network architectures emerged. We introduce IntelliSwAS approach for optimizing deep neural network architectures for a classification or regression task. Image classification was selected as the task, but IntelliSwAS is generic enough such that it could be successfully applied on other classification or regression problems. A particle swarm-based optimization algorithm is proposed for the automatic search for convolutional neural network architectures. The search technique is enhanced by a machine learning model (DAGRNN) which we designed for predicting the quality of the network architectures and thus increasing the performance of the algorithm. The proposed model is able to process data structured as directed acyclic graphs in general and we applied it specifically on network architectures. The network architecture that we discovered using IntelliSwAS surpassed 89.8% of the competing image classification models that we considered for comparison.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Deep learning is in a continuous evolution and many domains benefit from this substantial progress in the development of intelligent solutions. While this progress has been swift, there are more and more applications and the specific requirements of each application domain entails much work done by researchers to adapt existing models or create new ones. This traditional approach has produced many successful designs, but recently, automated methods of finding neural network architectures emerged. We introduce IntelliSwAS approach for optimizing deep neural network architectures for a classification or regression task. Image classification was selected as the task, but IntelliSwAS is generic enough such that it could be successfully applied on other classification or regression problems. A particle swarm-based optimization algorithm is proposed for the automatic search for convolutional neural network architectures. The search technique is enhanced by a machine learning model (DAGRNN) which we designed for predicting the quality of the network architectures and thus increasing the performance of the algorithm. The proposed model is able to process data structured as directed acyclic graphs in general and we applied it specifically on network architectures. The network architecture that we discovered using IntelliSwAS surpassed 89.8% of the competing image classification models that we considered for comparison.
1614.
Fei, Ke; Li, Qi; Zhu, Congcong
Non-technical losses detection using missing values’ pattern and neural architecture search Journal Article
In: International Journal of Electrical Power & Energy Systems, 134 , pp. 107410, 2022, ISSN: 0142-0615.
@article{FEI2022107410,
title = {Non-technical losses detection using missing values’ pattern and neural architecture search},
author = {Ke Fei and Qi Li and Congcong Zhu},
url = {https://www.sciencedirect.com/science/article/pii/S0142061521006499},
doi = {https://doi.org/10.1016/j.ijepes.2021.107410},
issn = {0142-0615},
year = {2022},
date = {2022-01-01},
journal = {International Journal of Electrical Power & Energy Systems},
volume = {134},
pages = {107410},
abstract = {The fast growth of non-technical loss (NTL) has gradually become one of the main concerns for distribution network operators (DNOs). Electricity theft which constitutes the main part of NTL not only brings losses to the DNOs, but also reduces the quality of the supply. A traditional detection method relies on utility workers’ experience and consumes a large amount of manpower. Thanks to the emerging of advanced metering infrastructure (AMI), utility companies can now collect detailed data reflecting consumers’ electricity usage, which enabled algorithms-based non-technical loss detection. The current data-based methods focus on the characteristics of electricity consumption thereby less efficient when dealing with rapidly changed electricity theft techniques. This article introduced a new data set, the location information of missing values, to improve the accuracy of non-technical loss detection. The relationship between missing values and electricity theft techniques is analyzed and a neural network model is built through neural architecture search (NAS). The improved model achieved an excellent Area Under Curve (AUC) value around 0.926 which verified the close link between missing values and electricity theft techniques. The nature of neural architecture search allows automatic model update which makes it a user-friendly tool even for engineers without any neural network expertise. A case study was carried out in which the missing value pattern was analyzed through Density-Based Spatial Clustering of Applications with Noise (DBSCAN) clustering algorithm.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
The fast growth of non-technical loss (NTL) has gradually become one of the main concerns for distribution network operators (DNOs). Electricity theft which constitutes the main part of NTL not only brings losses to the DNOs, but also reduces the quality of the supply. A traditional detection method relies on utility workers’ experience and consumes a large amount of manpower. Thanks to the emerging of advanced metering infrastructure (AMI), utility companies can now collect detailed data reflecting consumers’ electricity usage, which enabled algorithms-based non-technical loss detection. The current data-based methods focus on the characteristics of electricity consumption thereby less efficient when dealing with rapidly changed electricity theft techniques. This article introduced a new data set, the location information of missing values, to improve the accuracy of non-technical loss detection. The relationship between missing values and electricity theft techniques is analyzed and a neural network model is built through neural architecture search (NAS). The improved model achieved an excellent Area Under Curve (AUC) value around 0.926 which verified the close link between missing values and electricity theft techniques. The nature of neural architecture search allows automatic model update which makes it a user-friendly tool even for engineers without any neural network expertise. A case study was carried out in which the missing value pattern was analyzed through Density-Based Spatial Clustering of Applications with Noise (DBSCAN) clustering algorithm.
2021
1613.
Jixiang Li Shun Lu, Jianchao Tan
TNASP: A Transformer-based NAS Predictor with a Self-evolution Framework Inproceedings
In: 35th Conference on Neural Information Processing Systems (NeurIPS 2021), 2021.
@inproceedings{LuNeurIPS2021,
title = {TNASP: A Transformer-based NAS Predictor with a Self-evolution Framework},
author = {Shun Lu, Jixiang Li, Jianchao Tan, Sen Yang, Ji Liu},
url = {https://openreview.net/pdf?id=_aJnkoYKj6s},
year = {2021},
date = {2021-12-07},
booktitle = {35th Conference on Neural Information Processing Systems (NeurIPS 2021)},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
1612.
Gao, Yang; Zhang, Peng; Li, Zhao; Zhou, Chuan; Liu, Yongchao; Hu, Yue
Heterogeneous Graph Neural Architecture Search Inproceedings
In: 2021 IEEE International Conference on Data Mining (ICDM), 2021.
@inproceedings{GaoICDM2021,
title = {Heterogeneous Graph Neural Architecture Search},
author = {Yang Gao and Peng Zhang and Zhao Li and Chuan Zhou and Yongchao Liu and Yue Hu},
url = {https://www.researchgate.net/profile/Chuan-Zhou-3/publication/357512419_Heterogeneous_Graph_Neural_Architecture_Search/links/61d1889cb8305f7c4b19bd20/Heterogeneous-Graph-Neural-Architecture-Search.pdf},
year = {2021},
date = {2021-12-01},
urldate = {2021-12-01},
booktitle = {2021 IEEE International Conference on Data Mining (ICDM)},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
1611.
Tran, Linh Tam; Baea, Sung-Ho
Training-Free Hardware-Aware Neural Architecture Search with Reinforcement Learning Technical Report
2021.
@techreport{Tran2021,
title = {Training-Free Hardware-Aware Neural Architecture Search with Reinforcement Learning},
author = {Linh Tam Tran and Sung-Ho Baea},
url = {http://www.kpubs.org/article/articleDownload.kpubs?downType=pdf&articleANo=BSGHC3_2021_v26n7_855},
year = {2021},
date = {2021-12-01},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
1610.
Huang, Ziyan; Wang, Zehua; Yang, Zhikai; Gu, Lixu
AdwU-Net: Adaptive Depth and Width U-Net for Medical Image Segmentation by Differentiable Neural Architecture Search Technical Report
2021.
@techreport{Huang1MIDL2022,
title = {AdwU-Net: Adaptive Depth and Width U-Net for Medical Image Segmentation by Differentiable Neural Architecture Search},
author = {Ziyan Huang and Zehua Wang and Zhikai Yang and Lixu Gu},
url = {https://openreview.net/pdf?id=kF-d1SKWJpS},
year = {2021},
date = {2021-12-01},
urldate = {2022-02-01},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
1609.
Wang, Zifu; Blaschko, Matthew B.
MRF-NAS: Improving UNet with Neural Architecture Search for Medical Image Segmentation Technical Report
2021.
@techreport{WangPMUDL2021,
title = {MRF-NAS: Improving UNet with Neural Architecture Search for Medical Image Segmentation},
author = {Zifu Wang and Matthew B. Blaschko},
url = {https://openreview.net/pdf?id=M4FGQBtq7TS},
year = {2021},
date = {2021-12-01},
urldate = {2021-12-01},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
1608.
Mo, Shentong; Xia, Jingfei; Ren, Pinxu
Simulated Annealing for Neural Architecture Search Inproceedings
In: OPT2021: 13th Annual Workshop on Optimization for Machine Learning, 2021.
@inproceedings{Mo2021,
title = {Simulated Annealing for Neural Architecture Search},
author = {Shentong Mo and Jingfei Xia and Pinxu Ren},
url = {https://opt-ml.org/papers/2021/paper54.pdf},
year = {2021},
date = {2021-12-01},
urldate = {2021-12-01},
booktitle = {OPT2021: 13th Annual Workshop on Optimization for Machine Learning},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
1607.
Choe, Hyeokjun; Na, Byunggook; Mok, Jisoo; Yoon, Sungroh
Variance-stationary Differentiable NAS Inproceedings
In: British Machine Vision Conference (BMVC) 2021, 2021.
@inproceedings{Choe2021,
title = {Variance-stationary Differentiable NAS},
author = {Hyeokjun Choe and Byunggook Na and Jisoo Mok and Sungroh Yoon},
url = {https://www.bmvc2021-virtualconference.com/assets/papers/0549.pdf},
year = {2021},
date = {2021-12-01},
urldate = {2021-12-01},
booktitle = {British Machine Vision Conference (BMVC) 2021},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
1606.
Hao, Jie; Zhu, William
Layered Feature Representation for Differentiable Architecture Search Technical Report
2021.
@techreport{Hao2021c,
title = {Layered Feature Representation for Differentiable Architecture Search},
author = {Jie Hao and William Zhu},
url = {https://assets.researchsquare.com/files/rs-1086452/v1_covered.pdf?c=1638813118},
year = {2021},
date = {2021-12-01},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
1605.
Ye, Andre
Successful Neural Network Architecture Design Book Chapter
In: Apress, Berkeley, CA, 2021.
@inbook{Ye2021,
title = {Successful Neural Network Architecture Design},
author = {Andre Ye
},
url = {https://link.springer.com/chapter/10.1007/978-1-4842-7413-2_6},
year = {2021},
date = {2021-11-19},
urldate = {2021-11-19},
publisher = {Apress, Berkeley, CA},
keywords = {},
pubstate = {published},
tppubtype = {inbook}
}
1604.
Yan, C.; Chang, X.; Li, Z.; Guan, W.; Ge, Z.; Zhu, L.; Zheng, Q.
ZeroNAS: Differentiable Generative Adversarial Networks Search for Zero-Shot Learning Journal Article
In: IEEE Transactions on Pattern Analysis & Machine Intelligence, (01), pp. 1-1, 2021, ISSN: 1939-3539.
@article{9612044,
title = {ZeroNAS: Differentiable Generative Adversarial Networks Search for Zero-Shot Learning},
author = {C. Yan and X. Chang and Z. Li and W. Guan and Z. Ge and L. Zhu and Q. Zheng},
url = {https://www.computer.org/csdl/journal/tp/5555/01/09612044/1yrD49Irkmk},
doi = {10.1109/TPAMI.2021.3127346},
issn = {1939-3539},
year = {2021},
date = {2021-11-13},
urldate = {5555-11-01},
journal = {IEEE Transactions on Pattern Analysis & Machine Intelligence},
number = {01},
pages = {1-1},
publisher = {IEEE Computer Society},
address = {Los Alamitos, CA, USA},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
1603.
Li, Siyi; Sun, Yanan; Yen, Gary G; Zhang, Mengjie
Automatic Design of Convolutional Neural Network Architectures Under Resource Constraints Inproceedings
In: IEEE Trans Neural Netw Learn Syst ., 2021.
@inproceedings{LiTNNLS2021,
title = {Automatic Design of Convolutional Neural Network Architectures Under Resource Constraints},
author = {Siyi Li and Yanan Sun and Gary G Yen and Mengjie Zhang
},
url = {https://pubmed.ncbi.nlm.nih.gov/34752407/},
doi = {10.1109/TNNLS.2021.3123105},
year = {2021},
date = {2021-11-09},
booktitle = {IEEE Trans Neural Netw Learn Syst
.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
1602.
Wang, Shurun; Tang, Hao; Wang, Bin; Mo, Jia
A Novel Approach to Detecting Muscle Fatigue Based on sEMG by Using Neural Architecture Search Framework Inproceedings
In: IEEE Trans Neural Netw Learn Syst, 2021.
@inproceedings{WangTNNLS2021,
title = {A Novel Approach to Detecting Muscle Fatigue Based on sEMG by Using Neural Architecture Search Framework},
author = {Shurun Wang and Hao Tang and Bin Wang and Jia Mo
},
url = {https://pubmed.ncbi.nlm.nih.gov/34752412/},
doi = {10.1109/TNNLS.2021.3124330},
year = {2021},
date = {2021-11-09},
booktitle = { IEEE Trans Neural Netw Learn Syst},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
1601.
Liu, Zhengying
Automated Deep Learning : Principles and Practice PhD Thesis
2021.
@phdthesis{Liu2021,
title = {Automated Deep Learning : Principles and Practice},
author = {Zhengying Liu},
url = {https://tel.archives-ouvertes.fr/tel-03464519/document},
year = {2021},
date = {2021-11-01},
keywords = {},
pubstate = {published},
tppubtype = {phdthesis}
}
1600.
Nangi, Sharmila Reddy; Tyagi, Atharv; Mundra, Jay; Mukherjee, Sagnik; Snehal, Raj; Chhaya, Niyati; Garimella, Aparna
AUTOSUMM: Automatic Model Creation for Text Summarization Inproceedings
In: Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing, pp. 10162–10172, Ässociation for Computational Linguistics, Önline and Punta Cana, Dominican Republic, 2021.
@inproceedings{nangi-etal-2021-autosumm,
title = {AUTOSUMM: Automatic Model Creation for Text Summarization},
author = {Sharmila Reddy Nangi and Atharv Tyagi and Jay Mundra and Sagnik Mukherjee and Raj Snehal and Niyati Chhaya and Aparna Garimella},
url = {https://aclanthology.org/2021.emnlp-main.798},
year = {2021},
date = {2021-11-01},
urldate = {2021-11-01},
booktitle = {Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing},
pages = {10162--10172},
publisher = {Ässociation for Computational Linguistics},
address = {Önline and Punta Cana, Dominican Republic},
abstract = {Recent efforts to develop deep learning models for text generation tasks such as extractive and abstractive summarization have resulted in state-of-the-art performances on various datasets. However, obtaining the best model configuration for a given dataset requires an extensive knowledge of deep learning specifics like model architecture, tuning parameters etc., and is often extremely challenging for a non-expert. In this paper, we propose methods to automatically create deep learning models for the tasks of extractive and abstractive text summarization. Based on the recent advances in Automated Machine Learning and the success of large language models such as BERT and GPT-2 in encoding knowledge, we use a combination of Neural Architecture Search (NAS) and Knowledge Distillation (KD) techniques to perform model search and compression using the vast knowledge provided by these language models to develop smaller, customized models for any given dataset. We present extensive empirical results to illustrate the effectiveness of our model creation methods in terms of inference time and model size, while achieving near state-of-the-art performances in terms of accuracy across a range of datasets.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Recent efforts to develop deep learning models for text generation tasks such as extractive and abstractive summarization have resulted in state-of-the-art performances on various datasets. However, obtaining the best model configuration for a given dataset requires an extensive knowledge of deep learning specifics like model architecture, tuning parameters etc., and is often extremely challenging for a non-expert. In this paper, we propose methods to automatically create deep learning models for the tasks of extractive and abstractive text summarization. Based on the recent advances in Automated Machine Learning and the success of large language models such as BERT and GPT-2 in encoding knowledge, we use a combination of Neural Architecture Search (NAS) and Knowledge Distillation (KD) techniques to perform model search and compression using the vast knowledge provided by these language models to develop smaller, customized models for any given dataset. We present extensive empirical results to illustrate the effectiveness of our model creation methods in terms of inference time and model size, while achieving near state-of-the-art performances in terms of accuracy across a range of datasets.
1599.
Nguyen, N.; Chang, M. J.
CSNAS: Contrastive Self-supervised Learning Neural Architecture Search via Sequential Model-Based Optimization Journal Article
In: IEEE Transactions on Artificial Intelligence, 1 (01), pp. 1-1, 2021.
@article{9594655,
title = {CSNAS: Contrastive Self-supervised Learning Neural Architecture Search via Sequential Model-Based Optimization},
author = {N. Nguyen and M. J. Chang},
url = {https://www.computer.org/csdl/journal/ai/5555/01/09594655/1y5ZonWBo2c},
doi = {10.1109/TAI.2021.3121663},
year = {2021},
date = {2021-10-29},
urldate = {5555-10-01},
journal = {IEEE Transactions on Artificial Intelligence},
volume = {1},
number = {01},
pages = {1-1},
publisher = {IEEE Computer Society},
address = {Los Alamitos, CA, USA},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
1598.
Chen, Zerui; Huang, Yan; Yu, Hongyuan; Wang, Liang
Learning a Robust Part-Aware Monocular 3D Human Pose Estimator via Neural Architecture Search Journal Article
In: International Journal of Computer Vision (2021), 2021.
@article{nokey,
title = {Learning a Robust Part-Aware Monocular 3D Human Pose Estimator via Neural Architecture Search},
author = {Zerui Chen and Yan Huang and Hongyuan Yu and Liang Wang
},
url = {https://doi.org/10.1007/s11263-021-01525-0},
year = {2021},
date = {2021-10-26},
urldate = {2021-10-26},
journal = { International Journal of Computer Vision (2021)},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
1597.
Burrello, Alessio; Pagliari, Daniele Jahier; Risso, Matteo; Benatti, Simone; Macii, Enrico; Benini, Luca; Poncino, Massimo
Q-PPG: Energy-Efficient PPG-based Heart Rate Monitoring on Wearable Devices Journal Article
In: IEEE Trans Biomed Circuits Syst ., 2021.
@article{Burrello2201,
title = { Q-PPG: Energy-Efficient PPG-based Heart Rate Monitoring on Wearable Devices },
author = {Alessio Burrello and Daniele Jahier Pagliari and Matteo Risso and Simone Benatti and Enrico Macii and Luca Benini and Massimo Poncino
},
url = {https://pubmed.ncbi.nlm.nih.gov/34673496/},
doi = {10.1109/TBCAS.2021.3122017 },
year = {2021},
date = {2021-10-21},
journal = { IEEE Trans Biomed Circuits Syst .},
keywords = {},
pubstate = {published},
tppubtype = {article}
}