Maintained by Difan Deng and Marius Lindauer.
The following list considers papers related to neural architecture search. It is by no means complete. If you miss a paper on the list, please let us know.
Please note that although NAS methods steadily improve, the quality of empirical evaluations in this field are still lagging behind compared to other areas in machine learning, AI and optimization. We would therefore like to share some best practices for empirical evaluations of NAS methods, which we believe will facilitate sustained and measurable progress in the field. If you are interested in a teaser, please read our blog post or directly jump to our checklist.
Transformers have gained increasing popularity in different domains. For a comprehensive list of papers focusing on Neural Architecture Search for Transformer-Based spaces, the awesome-transformer-search repo is all you need.
2023
2439.
Cao, Yufan; Zhang, Tunhou; Wen, Wei; Yan, Feng; Li, Hai; Chen, Yiran
Farthest Greedy Path Sampling for Two-shot Recommender Search Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2310-20705,
title = {Farthest Greedy Path Sampling for Two-shot Recommender Search},
author = {Yufan Cao and Tunhou Zhang and Wei Wen and Feng Yan and Hai Li and Yiran Chen},
url = {https://doi.org/10.48550/arXiv.2310.20705},
doi = {10.48550/ARXIV.2310.20705},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2310.20705},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2438.
Ding, Li; Zoghi, Masrour; Tennenholtz, Guy; Karimzadehgan, Maryam
Ever Evolving Evaluator (EV3): Towards Flexible and Reliable Meta-Optimization for Knowledge Distillation Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2310-18893,
title = {Ever Evolving Evaluator (EV3): Towards Flexible and Reliable Meta-Optimization for Knowledge Distillation},
author = {Li Ding and Masrour Zoghi and Guy Tennenholtz and Maryam Karimzadehgan},
url = {https://doi.org/10.48550/arXiv.2310.18893},
doi = {10.48550/ARXIV.2310.18893},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2310.18893},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2437.
Zhang, Tunhou; Wen, Wei; Fedorov, Igor; Liu, Xi; Zhang, Buyun; Han, Fangqiu; Chen, Wen-Yen; Han, Yiping; Yan, Feng; Li, Hai; Chen, Yiran
DistDNAS: Search Efficient Feature Interactions within 2 Hours Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2311-00231,
title = {DistDNAS: Search Efficient Feature Interactions within 2 Hours},
author = {Tunhou Zhang and Wei Wen and Igor Fedorov and Xi Liu and Buyun Zhang and Fangqiu Han and Wen-Yen Chen and Yiping Han and Feng Yan and Hai Li and Yiran Chen},
url = {https://doi.org/10.48550/arXiv.2311.00231},
doi = {10.48550/ARXIV.2311.00231},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2311.00231},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2436.
Sun, Pingping; Fan, Shijie; Li, Shaochuan; Zhao, Yingwei; Lu, Chang; Wong, Ka-Chun; Li, Xiangtao
Automated exploitation of deep learning for cancer patient stratification across multiple types Journal Article
In: Bioinformatics, vol. 39, no. 11, pp. btad654, 2023, ISSN: 1367-4811.
@article{10.1093/bioinformatics/btad654,
title = {Automated exploitation of deep learning for cancer patient stratification across multiple types},
author = {Pingping Sun and Shijie Fan and Shaochuan Li and Yingwei Zhao and Chang Lu and Ka-Chun Wong and Xiangtao Li},
url = {https://doi.org/10.1093/bioinformatics/btad654},
doi = {10.1093/bioinformatics/btad654},
issn = {1367-4811},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {Bioinformatics},
volume = {39},
number = {11},
pages = {btad654},
abstract = {Recent frameworks based on deep learning have been developed to identify cancer subtypes from high-throughput gene expression profiles. Unfortunately, the performance of deep learning is highly dependent on its neural network architectures which are often hand-crafted with expertise in deep neural networks, meanwhile, the optimization and adjustment of the network are usually costly and time consuming.To address such limitations, we proposed a fully automated deep neural architecture search model for diagnosing consensus molecular subtypes from gene expression data (DNAS). The proposed model uses ant colony algorithm, one of the heuristic swarm intelligence algorithms, to search and optimize neural network architecture, and it can automatically find the optimal deep learning model architecture for cancer diagnosis in its search space. We validated DNAS on eight colorectal cancer datasets, achieving the average accuracy of 95.48%, the average specificity of 98.07%, and the average sensitivity of 96.24%, respectively. Without the loss of generality, we investigated the general applicability of DNAS further on other cancer types from different platforms including lung cancer and breast cancer, and DNAS achieved an area under the curve of 95% and 96%, respectively. In addition, we conducted gene ontology enrichment and pathological analysis to reveal interesting insights into cancer subtype identification and characterization across multiple cancer types.The source code and data can be downloaded from https://github.com/userd113/DNAS-main. And the web server of DNAS is publicly accessible at 119.45.145.120:5001.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Recent frameworks based on deep learning have been developed to identify cancer subtypes from high-throughput gene expression profiles. Unfortunately, the performance of deep learning is highly dependent on its neural network architectures which are often hand-crafted with expertise in deep neural networks, meanwhile, the optimization and adjustment of the network are usually costly and time consuming.To address such limitations, we proposed a fully automated deep neural architecture search model for diagnosing consensus molecular subtypes from gene expression data (DNAS). The proposed model uses ant colony algorithm, one of the heuristic swarm intelligence algorithms, to search and optimize neural network architecture, and it can automatically find the optimal deep learning model architecture for cancer diagnosis in its search space. We validated DNAS on eight colorectal cancer datasets, achieving the average accuracy of 95.48%, the average specificity of 98.07%, and the average sensitivity of 96.24%, respectively. Without the loss of generality, we investigated the general applicability of DNAS further on other cancer types from different platforms including lung cancer and breast cancer, and DNAS achieved an area under the curve of 95% and 96%, respectively. In addition, we conducted gene ontology enrichment and pathological analysis to reveal interesting insights into cancer subtype identification and characterization across multiple cancer types.The source code and data can be downloaded from https://github.com/userd113/DNAS-main. And the web server of DNAS is publicly accessible at 119.45.145.120:5001.
2435.
Li, Qing; Dai, Haixing; Lv, Jinglei; Zhao, Lin; Liu, Zhengliang; Wu, Zihao; Wu, Xia; Coles, Claire; Hu, Xiaoping; Liu, Tianming; Zhu, Dajiang
Individual Functional Network Abnormalities Mapping via Graph Representation-Based Neural Architecture Search Proceedings Article
In: Yang, Xiaochun; Suhartanto, Heru; Wang, Guoren; Wang, Bin; Jiang, Jing; Li, Bing; Zhu, Huaijie; Cui, Ningning (Ed.): Advanced Data Mining and Applications, pp. 79–91, Springer Nature Switzerland, Cham, 2023, ISBN: 978-3-031-46671-7.
@inproceedings{10.1007/978-3-031-46671-7_6,
title = {Individual Functional Network Abnormalities Mapping via Graph Representation-Based Neural Architecture Search},
author = {Qing Li and Haixing Dai and Jinglei Lv and Lin Zhao and Zhengliang Liu and Zihao Wu and Xia Wu and Claire Coles and Xiaoping Hu and Tianming Liu and Dajiang Zhu},
editor = {Xiaochun Yang and Heru Suhartanto and Guoren Wang and Bin Wang and Jing Jiang and Bing Li and Huaijie Zhu and Ningning Cui},
url = {https://link.springer.com/chapter/10.1007/978-3-031-46671-7_6},
isbn = {978-3-031-46671-7},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {Advanced Data Mining and Applications},
pages = {79–91},
publisher = {Springer Nature Switzerland},
address = {Cham},
abstract = {Prenatal alcohol exposure (PAE) has garnered increasing attention due to its detrimental effects on both neonates and expectant mothers. Recent research indicates that spatio-temporal functional brain networks (FBNs), derived from functional magnetic resonance imaging (fMRI), have the potential to reveal changes in PAE and Non-dysmorphic PAE (Non-Dys PAE) groups compared with healthy controls. However, current deep learning approaches for decomposing the FBNs are still limited to hand-crafted neural network architectures, which may not lead to optimal performance in identifying FBNs that better reveal differences between PAE and healthy controls. In this paper, we utilize a novel graph representation-based neural architecture search (GR-NAS) model to optimize the inner cell architecture of recurrent neural network (RNN) for decomposing the spatio-temporal FBNs and identifying the neuroimaging biomarkers of subtypes of PAE. Our optimized RNN cells with the GR-NAS model revealed that the functional activation decreased from healthy controls to Non-Dys PAE then to PAE groups. Our model provides a novel computational tool for the diagnosis of PAE, and uncovers the brain's functional mechanism in PAE.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Prenatal alcohol exposure (PAE) has garnered increasing attention due to its detrimental effects on both neonates and expectant mothers. Recent research indicates that spatio-temporal functional brain networks (FBNs), derived from functional magnetic resonance imaging (fMRI), have the potential to reveal changes in PAE and Non-dysmorphic PAE (Non-Dys PAE) groups compared with healthy controls. However, current deep learning approaches for decomposing the FBNs are still limited to hand-crafted neural network architectures, which may not lead to optimal performance in identifying FBNs that better reveal differences between PAE and healthy controls. In this paper, we utilize a novel graph representation-based neural architecture search (GR-NAS) model to optimize the inner cell architecture of recurrent neural network (RNN) for decomposing the spatio-temporal FBNs and identifying the neuroimaging biomarkers of subtypes of PAE. Our optimized RNN cells with the GR-NAS model revealed that the functional activation decreased from healthy controls to Non-Dys PAE then to PAE groups. Our model provides a novel computational tool for the diagnosis of PAE, and uncovers the brain's functional mechanism in PAE.
2434.
Yang, Shangshang; Zhen, Cheng; Tian, Ye; Ma, Haiping; Liu, Yuanchao; Zhang, Panpan; Zhang, Xingyi
Evolutionary Multi-Objective Neural Architecture Search for Generalized Cognitive Diagnosis Models Proceedings Article
In: 2023 5th International Conference on Data-driven Optimization of Complex Systems (DOCS), pp. 1-10, 2023.
@inproceedings{10294588,
title = {Evolutionary Multi-Objective Neural Architecture Search for Generalized Cognitive Diagnosis Models},
author = {Shangshang Yang and Cheng Zhen and Ye Tian and Haiping Ma and Yuanchao Liu and Panpan Zhang and Xingyi Zhang},
url = {https://ieeexplore.ieee.org/abstract/document/10294588},
doi = {10.1109/DOCS60977.2023.10294588},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {2023 5th International Conference on Data-driven Optimization of Complex Systems (DOCS)},
pages = {1-10},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
2433.
Shi, Huihong; Xu, Yang; Wang, Yuefei; Mao, Wendong; Wang, Zhongfeng
NASA-F: FPGA-Oriented Search and Acceleration for Multiplication-Reduced Hybrid Networks Journal Article
In: IEEE Transactions on Circuits and Systems I: Regular Papers, pp. 1-14, 2023.
@article{10308526,
title = {NASA-F: FPGA-Oriented Search and Acceleration for Multiplication-Reduced Hybrid Networks},
author = {Huihong Shi and Yang Xu and Yuefei Wang and Wendong Mao and Zhongfeng Wang},
doi = {10.1109/TCSI.2023.3327965},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {IEEE Transactions on Circuits and Systems I: Regular Papers},
pages = {1-14},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
2432.
Dooley, Samuel; Sukthanker, Rhea Sanjay; Dickerson, John P; White, Colin; Hutter, Frank; Goldblum, Micah
Rethinking Bias Mitigation: Fairer Architectures Make for Fairer Face Recognition Proceedings Article
In: Thirty-seventh Conference on Neural Information Processing Systems, 2023.
@inproceedings{<LineBreak>dooley2023rethinking,
title = {Rethinking Bias Mitigation: Fairer Architectures Make for Fairer Face Recognition},
author = {Samuel Dooley and Rhea Sanjay Sukthanker and John P Dickerson and Colin White and Frank Hutter and Micah Goldblum},
url = {https://openreview.net/forum?id=1vzF4zWQ1E},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {Thirty-seventh Conference on Neural Information Processing Systems},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
2431.
Niu, Ruicheng; Zhu, Ziyuan; Leng, Tao; Li, Chaofei; Liu, Yuxin; Meng, Dan
Improving Adversarial Robustness via Channel and Depth Compatibility Proceedings Article
In: Yang, Xiaochun; Suhartanto, Heru; Wang, Guoren; Wang, Bin; Jiang, Jing; Li, Bing; Zhu, Huaijie; Cui, Ningning (Ed.): Advanced Data Mining and Applications, pp. 137–149, Springer Nature Switzerland, Cham, 2023, ISBN: 978-3-031-46677-9.
@inproceedings{10.1007/978-3-031-46677-9_10,
title = {Improving Adversarial Robustness via Channel and Depth Compatibility},
author = {Ruicheng Niu and Ziyuan Zhu and Tao Leng and Chaofei Li and Yuxin Liu and Dan Meng},
editor = {Xiaochun Yang and Heru Suhartanto and Guoren Wang and Bin Wang and Jing Jiang and Bing Li and Huaijie Zhu and Ningning Cui},
isbn = {978-3-031-46677-9},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {Advanced Data Mining and Applications},
pages = {137–149},
publisher = {Springer Nature Switzerland},
address = {Cham},
abstract = {Several deep neural networks are vulnerable to adversarial samples that are imperceptible to humans. To address this challenge, a range of techniques have been proposed to design more robust model architectures. However, previous research has primarily focused on identifying atomic structures that are more resilient, while our work focuses on adapting the model in two spatial dimensions: width and depth. In this paper, we present a multi-objective neural architecture search (NAS) method that searches for optimal widths for different layers in spatial dimensions, referred to as DW-Net. We also propose a novel adversarial sample generation technique for one-shot that enhances search space diversity and promotes search efficiency. Our experimental results demonstrate that the proposed optimal neural architecture outperforms state-of-the-art NAS-based networks widely used in the literature in terms of adversarial accuracy, under different adversarial attacks and for different-sized tasks.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Several deep neural networks are vulnerable to adversarial samples that are imperceptible to humans. To address this challenge, a range of techniques have been proposed to design more robust model architectures. However, previous research has primarily focused on identifying atomic structures that are more resilient, while our work focuses on adapting the model in two spatial dimensions: width and depth. In this paper, we present a multi-objective neural architecture search (NAS) method that searches for optimal widths for different layers in spatial dimensions, referred to as DW-Net. We also propose a novel adversarial sample generation technique for one-shot that enhances search space diversity and promotes search efficiency. Our experimental results demonstrate that the proposed optimal neural architecture outperforms state-of-the-art NAS-based networks widely used in the literature in terms of adversarial accuracy, under different adversarial attacks and for different-sized tasks.
2430.
Passos, Dário; Mishra, Puneet
Deep Tutti Frutti: Exploring CNN architectures for dry matter prediction in fruit from multi-fruit near-infrared spectra Journal Article
In: Chemometrics and Intelligent Laboratory Systems, vol. 243, pp. 105023, 2023, ISSN: 0169-7439.
@article{PASSOS2023105023,
title = {Deep Tutti Frutti: Exploring CNN architectures for dry matter prediction in fruit from multi-fruit near-infrared spectra},
author = {Dário Passos and Puneet Mishra},
url = {https://www.sciencedirect.com/science/article/pii/S0169743923002733},
doi = {https://doi.org/10.1016/j.chemolab.2023.105023},
issn = {0169-7439},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {Chemometrics and Intelligent Laboratory Systems},
volume = {243},
pages = {105023},
abstract = {Convolutional Neural Networks (CNNs) have proven to be a valuable Deep Learning (DL) algorithm to model near-infrared spectral data in Chemometrics. However, optimizing CNN architectures and their associated hyperparameters for specific tasks is challenging. In this study, we explore the development of 1D-CNN architectures for the task of fruit dry matter (DM) estimation, testing various designs and optimization strategies to achieve a generic DL model that is robust against data fluctuations. The models are built using a multi-fruit data set (n = 2397) that includes NIR spectra of apples, kiwis, mangoes, and pears. The obtained CNN models are compared with PLS (taken as baseline), and to Locally Weighted PLS (LW-PLS) models. In general, the optimized CNN architectures obtained lower RMSEs (best RMSE = 0.605 %) than PLS (RMSE = 0.892 %) and LW-PLS (RMSE = 0.687 %) on a holdout test set. For this specific task, CNNs start outperforming PLS when the number of training samples is around 500. Furthermore, it is also shown how a global CNN model, trained on multi-fruit data, performs against PLS models of individual fruits in the sub-tasks of individual fruit DM prediction and generalization to an external mango data set. Overall, with proper architecture optimization, CNNs show strong performance and generalization for NIR-based dry matter estimation across diverse fruits.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Convolutional Neural Networks (CNNs) have proven to be a valuable Deep Learning (DL) algorithm to model near-infrared spectral data in Chemometrics. However, optimizing CNN architectures and their associated hyperparameters for specific tasks is challenging. In this study, we explore the development of 1D-CNN architectures for the task of fruit dry matter (DM) estimation, testing various designs and optimization strategies to achieve a generic DL model that is robust against data fluctuations. The models are built using a multi-fruit data set (n = 2397) that includes NIR spectra of apples, kiwis, mangoes, and pears. The obtained CNN models are compared with PLS (taken as baseline), and to Locally Weighted PLS (LW-PLS) models. In general, the optimized CNN architectures obtained lower RMSEs (best RMSE = 0.605 %) than PLS (RMSE = 0.892 %) and LW-PLS (RMSE = 0.687 %) on a holdout test set. For this specific task, CNNs start outperforming PLS when the number of training samples is around 500. Furthermore, it is also shown how a global CNN model, trained on multi-fruit data, performs against PLS models of individual fruits in the sub-tasks of individual fruit DM prediction and generalization to an external mango data set. Overall, with proper architecture optimization, CNNs show strong performance and generalization for NIR-based dry matter estimation across diverse fruits.
2429.
Schrodi, Simon; Stoll, Danny; Ru, Binxin; Sukthanker, Rhea; Brox, Thomas; Hutter, Frank
Construction of Hierarchical Neural Architecture Search Spaces based on Context-free Grammars Proceedings Article
In: Advances in Neural Information Processing Systems, 2023.
@inproceedings{schrodi2023construction,
title = {Construction of Hierarchical Neural Architecture Search Spaces based on Context-free Grammars},
author = {Simon Schrodi and Danny Stoll and Binxin Ru and Rhea Sukthanker and Thomas Brox and Frank Hutter},
url = {https://openreview.net/pdf?id=Hpt1i5j6wh},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {Advances in Neural Information Processing Systems},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
2428.
Sinha, Nilotpal; Shabayek, Abd El Rahman; Kacem, Anis; Rostami, Peyman; Shneider, Carl; Aouada, Djamila
Hardware Aware Evolutionary Neural Architecture Search using Representation Similarity Metric Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2311-03923,
title = {Hardware Aware Evolutionary Neural Architecture Search using Representation Similarity Metric},
author = {Nilotpal Sinha and Abd El Rahman Shabayek and Anis Kacem and Peyman Rostami and Carl Shneider and Djamila Aouada},
url = {https://doi.org/10.48550/arXiv.2311.03923},
doi = {10.48550/ARXIV.2311.03923},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2311.03923},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2427.
McKnight, Shaun; MacKinnon, Christopher; Pierce, S. Gareth; Mohseni, Ehsan; Tunukovic, Vedran; MacLeod, Charles N.; Vithanage, Randika K. W.; OHare, Tom
3-Dimensional residual neural architecture search for ultrasonic defect detection Technical Report
2023.
@techreport{mcknight20233dimensional,
title = {3-Dimensional residual neural architecture search for ultrasonic defect detection},
author = {Shaun McKnight and Christopher MacKinnon and S. Gareth Pierce and Ehsan Mohseni and Vedran Tunukovic and Charles N. MacLeod and Randika K. W. Vithanage and Tom OHare},
url = {https://arxiv.org/abs/2311.01867},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2426.
Li, Yueyang; Zhang, Qican
NAS-assisted single-shot real-time 3D measurement Proceedings Article
In: Shao, Xiaopeng (Ed.): Third International Computing Imaging Conference (CITA 2023), pp. 129210S, International Society for Optics and Photonics SPIE, 2023.
@inproceedings{10.1117/12.2688044,
title = {NAS-assisted single-shot real-time 3D measurement},
author = {Yueyang Li and Qican Zhang},
editor = {Xiaopeng Shao},
url = {https://doi.org/10.1117/12.2688044},
doi = {10.1117/12.2688044},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {Third International Computing Imaging Conference (CITA 2023)},
volume = {12921},
pages = {129210S},
publisher = {SPIE},
organization = {International Society for Optics and Photonics},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
2425.
Prezja, Fabi; Annala, Leevi; Kiiskinen, Sampsa; Lahtinen, Suvi; Ojala, Timo
2023.
@techreport{unknownb,
title = {Adaptive Variance Thresholding: A Novel Approach to Improve Existing Deep Transfer Vision Models and Advance Automatic Knee-Joint Osteoarthritis Classification},
author = {Fabi Prezja and Leevi Annala and Sampsa Kiiskinen and Suvi Lahtinen and Timo Ojala},
url = {https://www.researchgate.net/profile/Fabi-Prezja/publication/375336306_Adaptive_Variance_Thresholding_A_Novel_Approach_to_Improve_Existing_Deep_Transfer_Vision_Models_and_Advance_Automatic_Knee-Joint_Osteoarthritis_Classification/links/6546cda8ce88b87031c4e5ad/Adaptive-Variance-Thresholding-A-Novel-Approach-to-Improve-Existing-Deep-Transfer-Vision-Models-and-Advance-Automatic-Knee-Joint-Osteoarthritis-Classification.pdf},
doi = {10.13140/RG.2.2.10736.02566},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2424.
Hou, Jia; Zhang, Jingyu; Chen, Qi; Xiang, Siwei; Meng, Yishuo; Wang, Jianfei; Lu, Cimang; Yang, Chen
In: Information, vol. 14, no. 11, 2023, ISSN: 2078-2489.
@article{info14110604,
title = {POSS-CNN: An Automatically Generated Convolutional Neural Network with Precision and Operation Separable Structure Aiming at Target Recognition and Detection},
author = {Jia Hou and Jingyu Zhang and Qi Chen and Siwei Xiang and Yishuo Meng and Jianfei Wang and Cimang Lu and Chen Yang},
url = {https://www.mdpi.com/2078-2489/14/11/604},
doi = {10.3390/info14110604},
issn = {2078-2489},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {Information},
volume = {14},
number = {11},
abstract = {Artificial intelligence is changing and influencing our world. As one of the main algorithms in the field of artificial intelligence, convolutional neural networks (CNNs) have developed rapidly in recent years. Especially after the emergence of NASNet, CNNs have gradually pushed the idea of AutoML to the public’s attention, and large numbers of new structures designed by automatic searches are appearing. These networks are usually based on reinforcement learning and evolutionary learning algorithms. However, sometimes, the blocks of these networks are complex, and there is no small model for simpler tasks. Therefore, this paper proposes POSS-CNN aiming at target recognition and detection, which employs a multi-branch CNN structure with PSNC and a method of automatic parallel selection for super parameters based on a multi-branch CNN structure. Moreover, POSS-CNN can be broken up. By choosing a single branch or the combination of two branches as the “benchmark”, as well as the overall POSS-CNN, we can achieve seven models with different precision and operations. The test accuracy of POSS-CNN for a recognition task tested on a CIFAR10 dataset can reach 86.4%, which is equivalent to AlexNet and VggNet, but the operation and parameters of the whole model in this paper are 45.9% and 45.8% of AlexNet, and 29.5% and 29.4% of VggNet. The mAP of POSS-CNN for a detection task tested on the LSVH dataset is 45.8, inferior to the 62.3 of YOLOv3. However, compared with YOLOv3, the operation and parameters of the model in this paper are reduced by 57.4% and 15.6%, respectively. After being accelerated by WRA, POSS-CNN for a detection task tested on an LSVH dataset can achieve 27 fps, and the energy efficiency is 0.42 J/f, which is 5 times and 96.6 times better than GPU 2080Ti in performance and energy efficiency, respectively.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Artificial intelligence is changing and influencing our world. As one of the main algorithms in the field of artificial intelligence, convolutional neural networks (CNNs) have developed rapidly in recent years. Especially after the emergence of NASNet, CNNs have gradually pushed the idea of AutoML to the public’s attention, and large numbers of new structures designed by automatic searches are appearing. These networks are usually based on reinforcement learning and evolutionary learning algorithms. However, sometimes, the blocks of these networks are complex, and there is no small model for simpler tasks. Therefore, this paper proposes POSS-CNN aiming at target recognition and detection, which employs a multi-branch CNN structure with PSNC and a method of automatic parallel selection for super parameters based on a multi-branch CNN structure. Moreover, POSS-CNN can be broken up. By choosing a single branch or the combination of two branches as the “benchmark”, as well as the overall POSS-CNN, we can achieve seven models with different precision and operations. The test accuracy of POSS-CNN for a recognition task tested on a CIFAR10 dataset can reach 86.4%, which is equivalent to AlexNet and VggNet, but the operation and parameters of the whole model in this paper are 45.9% and 45.8% of AlexNet, and 29.5% and 29.4% of VggNet. The mAP of POSS-CNN for a detection task tested on the LSVH dataset is 45.8, inferior to the 62.3 of YOLOv3. However, compared with YOLOv3, the operation and parameters of the model in this paper are reduced by 57.4% and 15.6%, respectively. After being accelerated by WRA, POSS-CNN for a detection task tested on an LSVH dataset can achieve 27 fps, and the energy efficiency is 0.42 J/f, which is 5 times and 96.6 times better than GPU 2080Ti in performance and energy efficiency, respectively.
2423.
Wu, Meng-Ting; Tsai, Chun-Wei
Training-free neural architecture search: A review Journal Article
In: ICT Express, 2023, ISSN: 2405-9595.
@article{WU2023,
title = {Training-free neural architecture search: A review},
author = {Meng-Ting Wu and Chun-Wei Tsai},
url = {https://www.sciencedirect.com/science/article/pii/S2405959523001443},
doi = {https://doi.org/10.1016/j.icte.2023.11.001},
issn = {2405-9595},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {ICT Express},
abstract = {The goal of neural architecture search (NAS) is to either downsize the neural architecture and model of a deep neural network (DNN), adjust a neural architecture to improve its end result, or even speed up the whole training process. Such improvements make it possible to generate or install the model of a DNN on a small device, such as a device of internet of things or wireless sensor network. Because most NAS algorithms are time-consuming, finding out a way to reduce their computation costs has now become a critical research issue. The training-free method (also called the zero-shot learning) provides an alternative way to estimate how good a neural architecture is more efficiently during the process of NAS by using a lightweight score function instead of a general training process to avoid incurring heavy costs. This paper starts with a brief discussion of DNN and NAS, followed by a brief review of both model-dependent and model-independent training-free score functions. A brief introduction to the search algorithms and benchmarks that were widely used in a training-free NAS will also be given in this paper. The changes, potential, open issues, and future trends of this research topic are then addressed in the end of this paper.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
The goal of neural architecture search (NAS) is to either downsize the neural architecture and model of a deep neural network (DNN), adjust a neural architecture to improve its end result, or even speed up the whole training process. Such improvements make it possible to generate or install the model of a DNN on a small device, such as a device of internet of things or wireless sensor network. Because most NAS algorithms are time-consuming, finding out a way to reduce their computation costs has now become a critical research issue. The training-free method (also called the zero-shot learning) provides an alternative way to estimate how good a neural architecture is more efficiently during the process of NAS by using a lightweight score function instead of a general training process to avoid incurring heavy costs. This paper starts with a brief discussion of DNN and NAS, followed by a brief review of both model-dependent and model-independent training-free score functions. A brief introduction to the search algorithms and benchmarks that were widely used in a training-free NAS will also be given in this paper. The changes, potential, open issues, and future trends of this research topic are then addressed in the end of this paper.
2422.
Dong, Dong; Jiang, Hongxu; Wei, Xuekai; Song, Yanfei; Zhuang, Xu; Wang, Jason
ETNAS: An energy consumption task-driven neural architecture search Journal Article
In: Sustainable Computing: Informatics and Systems, vol. 40, pp. 100926, 2023, ISSN: 2210-5379.
@article{DONG2023100926,
title = {ETNAS: An energy consumption task-driven neural architecture search},
author = {Dong Dong and Hongxu Jiang and Xuekai Wei and Yanfei Song and Xu Zhuang and Jason Wang},
url = {https://www.sciencedirect.com/science/article/pii/S2210537923000811},
doi = {https://doi.org/10.1016/j.suscom.2023.100926},
issn = {2210-5379},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {Sustainable Computing: Informatics and Systems},
volume = {40},
pages = {100926},
abstract = {Neural Architecture Search (NAS) is crucial in the field of sustainable computing as it facilitates the development of highly efficient and effective neural networks. However, it cannot automate the deployment of neural networks to accommodate specific hardware resources and task requirements. This paper introduces ETNAS, which is a hardware-aware multi-objective optimal neural network architecture search algorithm based on the differentiable neural network architecture search method (DARTS). The algorithm searches for a lower-power neural network architecture with guaranteed inference accuracy by modifying the loss function of the differentiable neural network architecture search. We modify the dense network in DARTS to simultaneously search for networks with a lower memory footprint, enabling them to run on memory-constrained edge-end devices. We collected data on the power consumption and time consumption of numerous common operators on FPGA and Domain-Specific Architectures (DSA). The experimental results demonstrate that ETNAS achieves comparable accuracy performance and time efficiency while consuming less power compared to state-of-the-art algorithms, thereby validating its effectiveness in practical applications and contributing to the reduction of carbon emissions in intelligent cyber–physical systems (ICPS) edge computing inference.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Neural Architecture Search (NAS) is crucial in the field of sustainable computing as it facilitates the development of highly efficient and effective neural networks. However, it cannot automate the deployment of neural networks to accommodate specific hardware resources and task requirements. This paper introduces ETNAS, which is a hardware-aware multi-objective optimal neural network architecture search algorithm based on the differentiable neural network architecture search method (DARTS). The algorithm searches for a lower-power neural network architecture with guaranteed inference accuracy by modifying the loss function of the differentiable neural network architecture search. We modify the dense network in DARTS to simultaneously search for networks with a lower memory footprint, enabling them to run on memory-constrained edge-end devices. We collected data on the power consumption and time consumption of numerous common operators on FPGA and Domain-Specific Architectures (DSA). The experimental results demonstrate that ETNAS achieves comparable accuracy performance and time efficiency while consuming less power compared to state-of-the-art algorithms, thereby validating its effectiveness in practical applications and contributing to the reduction of carbon emissions in intelligent cyber–physical systems (ICPS) edge computing inference.
2421.
Tang, Siao; Wang, Xin; Chen, Hong; Guan, Chaoyu; Tang, Yansong; Zhu, Wenwu
Lightweight Diffusion Models with Distillation-Based Block Neural Architecture Search Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2311-04950,
title = {Lightweight Diffusion Models with Distillation-Based Block Neural Architecture Search},
author = {Siao Tang and Xin Wang and Hong Chen and Chaoyu Guan and Yansong Tang and Wenwu Zhu},
url = {https://doi.org/10.48550/arXiv.2311.04950},
doi = {10.48550/ARXIV.2311.04950},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2311.04950},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2420.
Li, Yuke; Baik, Jiwon; Rahman, Md Marufi; Anagnostopoulos, Iraklis; Li, Ruopu; Shu, Tong
Pareto Optimization of CNN Models via Hardware-Aware Neural Architecture Search for Drainage Crossing Classification on Resource-Limited Devices Proceedings Article
In: Proceedings of the SC '23 Workshops of The International Conference on High Performance Computing, Network, Storage, and Analysis, pp. 1767–1775, Association for Computing Machinery, Denver, CO, USA, 2023, ISBN: 9798400707858.
@inproceedings{10.1145/3624062.3624258,
title = {Pareto Optimization of CNN Models via Hardware-Aware Neural Architecture Search for Drainage Crossing Classification on Resource-Limited Devices},
author = {Yuke Li and Jiwon Baik and Md Marufi Rahman and Iraklis Anagnostopoulos and Ruopu Li and Tong Shu},
url = {https://doi.org/10.1145/3624062.3624258},
doi = {10.1145/3624062.3624258},
isbn = {9798400707858},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {Proceedings of the SC '23 Workshops of The International Conference on High Performance Computing, Network, Storage, and Analysis},
pages = {1767–1775},
publisher = {Association for Computing Machinery},
address = {Denver, CO, USA},
series = {SC-W '23},
abstract = {Embedded devices, constrained by limited memory and processors, require deep learning models to be tailored to their specifications. This research explores customized model architectures for classifying drainage crossing images. Building on the foundational ResNet-18, this paper aims to maximize prediction accuracy, reduce memory size, and minimize inference latency. Various configurations were systematically probed by leveraging hardware-aware neural architecture search, accumulating 1,717 experimental results over six benchmarking variants. The experimental data analysis, enhanced by nn-Meter, provided a comprehensive understanding of inference latency across four different predictors. Significantly, a Pareto front analysis with three objectives of accuracy, latency, and memory resulted in five non-dominated solutions. These standout models showcased efficiency while retaining accuracy, offering a compelling alternative to the conventional ResNet-18 when deployed in resource-constrained environments. The paper concludes by highlighting insights drawn from the results and suggesting avenues for future exploration.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Embedded devices, constrained by limited memory and processors, require deep learning models to be tailored to their specifications. This research explores customized model architectures for classifying drainage crossing images. Building on the foundational ResNet-18, this paper aims to maximize prediction accuracy, reduce memory size, and minimize inference latency. Various configurations were systematically probed by leveraging hardware-aware neural architecture search, accumulating 1,717 experimental results over six benchmarking variants. The experimental data analysis, enhanced by nn-Meter, provided a comprehensive understanding of inference latency across four different predictors. Significantly, a Pareto front analysis with three objectives of accuracy, latency, and memory resulted in five non-dominated solutions. These standout models showcased efficiency while retaining accuracy, offering a compelling alternative to the conventional ResNet-18 when deployed in resource-constrained environments. The paper concludes by highlighting insights drawn from the results and suggesting avenues for future exploration.
2419.
Hsieh, Jun-Wei; Chou, Cheng-Han; Chang, Ming-Ching; Chen, Ping-Yang; Santra, Santanu; Huang, Chih-Sheng
Mean-Shift Based Differentiable Architecture Search Journal Article
In: IEEE Transactions on Artificial Intelligence, pp. 1-11, 2023.
@article{10310657,
title = {Mean-Shift Based Differentiable Architecture Search},
author = {Jun-Wei Hsieh and Cheng-Han Chou and Ming-Ching Chang and Ping-Yang Chen and Santanu Santra and Chih-Sheng Huang},
url = {https://ieeexplore.ieee.org/abstract/document/10310657},
doi = {10.1109/TAI.2023.3329792},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {IEEE Transactions on Artificial Intelligence},
pages = {1-11},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
2418.
Chang, Chi-Chih; Sung, Yuan-Yao; Yu, Shixing; Huang, Ning-Chi; Marculescu, Diana; Wu, Kai-Chiang
FLORA: Fine-grained Low-Rank Architecture Search for Vision Transformer Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2311-03912,
title = {FLORA: Fine-grained Low-Rank Architecture Search for Vision Transformer},
author = {Chi-Chih Chang and Yuan-Yao Sung and Shixing Yu and Ning-Chi Huang and Diana Marculescu and Kai-Chiang Wu},
url = {https://doi.org/10.48550/arXiv.2311.03912},
doi = {10.48550/ARXIV.2311.03912},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2311.03912},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2417.
Chang, Chen-Chia; Pan, Jingyu; Xie, Zhiyao; Zhang, Tunhou; Hu, Jiang; Chen, Yiran
Towards Fully Automated Machine Learning for Routability Estimator Development Journal Article
In: IEEE Transactions on Computer-Aided Design of Integrated Circuits and Systems, pp. 1-1, 2023.
@article{10310247,
title = {Towards Fully Automated Machine Learning for Routability Estimator Development},
author = {Chen-Chia Chang and Jingyu Pan and Zhiyao Xie and Tunhou Zhang and Jiang Hu and Yiran Chen},
url = {https://ieeexplore.ieee.org/abstract/document/10310247},
doi = {10.1109/TCAD.2023.3330818},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {IEEE Transactions on Computer-Aided Design of Integrated Circuits and Systems},
pages = {1-1},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
2416.
An, Taegun; Joo, Changhee
CycleGANAS: Differentiable Neural Architecture Search for CycleGAN Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2311-07162,
title = {CycleGANAS: Differentiable Neural Architecture Search for CycleGAN},
author = {Taegun An and Changhee Joo},
url = {https://doi.org/10.48550/arXiv.2311.07162},
doi = {10.48550/ARXIV.2311.07162},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2311.07162},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2415.
Ji, Junzhong; Wang, Xingyu
Convolutional architecture search based on particle swarm algorithm for functional brain network classification Journal Article
In: Applied Soft Computing, vol. 149, pp. 111049, 2023, ISSN: 1568-4946.
@article{JI2023111049,
title = {Convolutional architecture search based on particle swarm algorithm for functional brain network classification},
author = {Junzhong Ji and Xingyu Wang},
url = {https://www.sciencedirect.com/science/article/pii/S1568494623010670},
doi = {https://doi.org/10.1016/j.asoc.2023.111049},
issn = {1568-4946},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {Applied Soft Computing},
volume = {149},
pages = {111049},
abstract = {The functional brain network (FBN) classification based on convolutional neural networks (CNN) is of great significance for discovery and diagnosis of brain diseases, and has attracted increasing attention. However, all the CNN architectures of current studies mainly depend on hand-crafted, which are labor-intensive and unreliable. To solve it, we propose a neural architecture search (NAS) method based on particle swarm optimization, to automatically design the CNN architecture for FBN classification. Specifically, this method includes three phases, namely the individual expression phase, the individual evaluation phase, and the individual update phase. In the first phase, we treat the neural architecture as the individual in particle swarm. The individual vector consists of six elements, and the value of each element represents the number of a special convolution operation. The six special convolution operations can effectively extract brain network multilevel topological features. In the second phase, we propose a novel surrogate-assisted predictor to evaluate the fitness of the individuals more efficiently. In the last phase, we apply the predicted fitness to acquire the historical optimum of each individual and the global optimum of the population, and use them to update all individuals in the particle swarm. The second and third phases are repeatedly performed until the end condition is met. Experiments on benchmark datasets demonstrate that the CNN architecture searched by our method achieves better classification performance than state-of-the-art hand-crafted CNN architectures.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
The functional brain network (FBN) classification based on convolutional neural networks (CNN) is of great significance for discovery and diagnosis of brain diseases, and has attracted increasing attention. However, all the CNN architectures of current studies mainly depend on hand-crafted, which are labor-intensive and unreliable. To solve it, we propose a neural architecture search (NAS) method based on particle swarm optimization, to automatically design the CNN architecture for FBN classification. Specifically, this method includes three phases, namely the individual expression phase, the individual evaluation phase, and the individual update phase. In the first phase, we treat the neural architecture as the individual in particle swarm. The individual vector consists of six elements, and the value of each element represents the number of a special convolution operation. The six special convolution operations can effectively extract brain network multilevel topological features. In the second phase, we propose a novel surrogate-assisted predictor to evaluate the fitness of the individuals more efficiently. In the last phase, we apply the predicted fitness to acquire the historical optimum of each individual and the global optimum of the population, and use them to update all individuals in the particle swarm. The second and third phases are repeatedly performed until the end condition is met. Experiments on benchmark datasets demonstrate that the CNN architecture searched by our method achieves better classification performance than state-of-the-art hand-crafted CNN architectures.
2414.
Wen, Wei; Liu, Kuang-Hung; Fedorov, Igor; Zhang, Xin; Yin, Hang; Chu, Weiwei; Hassani, Kaveh; Sun, Mengying; Liu, Jiang; Wang, Xu; Jiang, Lin; Chen, Yuxin; Zhang, Buyun; Liu, Xi; Cheng, Dehua; Chen, Zhengxing; Zhao, Guang; Han, Fangqiu; Yang, Jiyan; Hao, Yuchen; Xiong, Liang; Chen, Wen-Yen
Rankitect: Ranking Architecture Search Battling World-class Engineers at Meta Scale Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2311-08430,
title = {Rankitect: Ranking Architecture Search Battling World-class Engineers at Meta Scale},
author = {Wei Wen and Kuang-Hung Liu and Igor Fedorov and Xin Zhang and Hang Yin and Weiwei Chu and Kaveh Hassani and Mengying Sun and Jiang Liu and Xu Wang and Lin Jiang and Yuxin Chen and Buyun Zhang and Xi Liu and Dehua Cheng and Zhengxing Chen and Guang Zhao and Fangqiu Han and Jiyan Yang and Yuchen Hao and Liang Xiong and Wen-Yen Chen},
url = {https://doi.org/10.48550/arXiv.2311.08430},
doi = {10.48550/ARXIV.2311.08430},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2311.08430},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2413.
Wang, Zhenrong; Li, Bin; Li, Weifeng; Niu, Shuanlong; Miao, Wang; Niu, Tongzhi
NAS-ASDet: An Adaptive Design Method for Surface Defect Detection Network using Neural Architecture Search Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2311-10952,
title = {NAS-ASDet: An Adaptive Design Method for Surface Defect Detection Network using Neural Architecture Search},
author = {Zhenrong Wang and Bin Li and Weifeng Li and Shuanlong Niu and Wang Miao and Tongzhi Niu},
url = {https://doi.org/10.48550/arXiv.2311.10952},
doi = {10.48550/ARXIV.2311.10952},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2311.10952},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2412.
Hu, Yiming; Chu, Xiangxiang; Zhang, Bo
Masked Autoencoders Are Robust Neural Architecture Search Learners Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2311-12086,
title = {Masked Autoencoders Are Robust Neural Architecture Search Learners},
author = {Yiming Hu and Xiangxiang Chu and Bo Zhang},
url = {https://doi.org/10.48550/arXiv.2311.12086},
doi = {10.48550/ARXIV.2311.12086},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2311.12086},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2411.
Han, Xiaoyu; Li, Chenyu; Wang, Zifan; Liu, Guohua
NDARTS: A Differentiable Architecture Search Based on the Neumann Series Journal Article
In: Algorithms, vol. 16, no. 12, 2023, ISSN: 1999-4893.
@article{a16120536,
title = {NDARTS: A Differentiable Architecture Search Based on the Neumann Series},
author = {Xiaoyu Han and Chenyu Li and Zifan Wang and Guohua Liu},
url = {https://www.mdpi.com/1999-4893/16/12/536},
doi = {10.3390/a16120536},
issn = {1999-4893},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {Algorithms},
volume = {16},
number = {12},
abstract = {Neural architecture search (NAS) has shown great potential in discovering powerful and flexible network models, becoming an important branch of automatic machine learning (AutoML). Although search methods based on reinforcement learning and evolutionary algorithms can find high-performance architectures, these search methods typically require hundreds of GPU days. Unlike searching in a discrete search space based on reinforcement learning and evolutionary algorithms, the differentiable neural architecture search (DARTS) continuously relaxes the search space, allowing for optimization using gradient-based methods. Based on DARTS, we propose NDARTS in this article. The new algorithm uses the Implicit Function Theorem and the Neumann series to approximate the hyper-gradient, which obtains better results than DARTS. In the simulation experiment, an ablation experiment was carried out to study the influence of the different parameters on the NDARTS algorithm and to determine the optimal weight, then the best performance of the NDARTS algorithm was searched for in the DARTS search space and the NAS-BENCH-201 search space. Compared with other NAS algorithms, the results showed that NDARTS achieved excellent results on the CIFAR-10, CIFAR-100, and ImageNet datasets, and was an effective neural architecture search algorithm.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Neural architecture search (NAS) has shown great potential in discovering powerful and flexible network models, becoming an important branch of automatic machine learning (AutoML). Although search methods based on reinforcement learning and evolutionary algorithms can find high-performance architectures, these search methods typically require hundreds of GPU days. Unlike searching in a discrete search space based on reinforcement learning and evolutionary algorithms, the differentiable neural architecture search (DARTS) continuously relaxes the search space, allowing for optimization using gradient-based methods. Based on DARTS, we propose NDARTS in this article. The new algorithm uses the Implicit Function Theorem and the Neumann series to approximate the hyper-gradient, which obtains better results than DARTS. In the simulation experiment, an ablation experiment was carried out to study the influence of the different parameters on the NDARTS algorithm and to determine the optimal weight, then the best performance of the NDARTS algorithm was searched for in the DARTS search space and the NAS-BENCH-201 search space. Compared with other NAS algorithms, the results showed that NDARTS achieved excellent results on the CIFAR-10, CIFAR-100, and ImageNet datasets, and was an effective neural architecture search algorithm.
2410.
Li, Xiangyi; Guo, Jiajia; Wen, Chao-Kai; Jin, Shi
Auto-CsiNet: Scenario-customized Automatic Neural Network Architecture Generation for Massive MIMO CSI Feedback Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2311-15950,
title = {Auto-CsiNet: Scenario-customized Automatic Neural Network Architecture Generation for Massive MIMO CSI Feedback},
author = {Xiangyi Li and Jiajia Guo and Chao-Kai Wen and Shi Jin},
url = {https://doi.org/10.48550/arXiv.2311.15950},
doi = {10.48550/ARXIV.2311.15950},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2311.15950},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2409.
Hafiz, Faizal M. F.; Broekaert, Jan; Swain, Akshya
Evolution of Neural Architectures for Financial Forecasting: A Note on Data Incompatibility during Crisis Periods Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2311-14604,
title = {Evolution of Neural Architectures for Financial Forecasting: A Note on Data Incompatibility during Crisis Periods},
author = {Faizal M. F. Hafiz and Jan Broekaert and Akshya Swain},
url = {https://doi.org/10.48550/arXiv.2311.14604},
doi = {10.48550/ARXIV.2311.14604},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2311.14604},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2408.
Wei, Zimian; Pan, Hengyue; Li, Lujun; Dong, Peijie; Tian, Zhiliang; Niu, Xin; Li, Dongsheng
TVT: Training-Free Vision Transformer Search on Tiny Datasets Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2311-14337,
title = {TVT: Training-Free Vision Transformer Search on Tiny Datasets},
author = {Zimian Wei and Hengyue Pan and Lujun Li and Peijie Dong and Zhiliang Tian and Xin Niu and Dongsheng Li},
url = {https://doi.org/10.48550/arXiv.2311.14337},
doi = {10.48550/ARXIV.2311.14337},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2311.14337},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2407.
Herberg, Evelyn; Herzog, Roland; Köhne, Frederik; Kreis, Leonie; Schiela, Anton
Sensitivity-Based Layer Insertion for Residual and Feedforward Neural Networks Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2311-15995,
title = {Sensitivity-Based Layer Insertion for Residual and Feedforward Neural Networks},
author = {Evelyn Herberg and Roland Herzog and Frederik Köhne and Leonie Kreis and Anton Schiela},
url = {https://doi.org/10.48550/arXiv.2311.15995},
doi = {10.48550/ARXIV.2311.15995},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2311.15995},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2406.
Haq, Ijaz Ul; Lee, Byung Suk
TransNAS-TSAD: Harnessing Transformers for Multi-Objective Neural Architecture Search in Time Series Anomaly Detection Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2311-18061,
title = {TransNAS-TSAD: Harnessing Transformers for Multi-Objective Neural Architecture Search in Time Series Anomaly Detection},
author = {Ijaz Ul Haq and Byung Suk Lee},
url = {https://doi.org/10.48550/arXiv.2311.18061},
doi = {10.48550/ARXIV.2311.18061},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2311.18061},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2405.
Loya, Hrushikesh; Dudziak, Lukasz; Mehrotra, Abhinav; Lee, Royson; Fernández-Marqués, Javier; Lane, Nicholas D.; Wen, Hongkai
How Much Is Hidden in the NAS Benchmarks? Few-Shot Adaptation of a NAS Predictor Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2311-18451,
title = {How Much Is Hidden in the NAS Benchmarks? Few-Shot Adaptation of a NAS Predictor},
author = {Hrushikesh Loya and Lukasz Dudziak and Abhinav Mehrotra and Royson Lee and Javier Fernández-Marqués and Nicholas D. Lane and Hongkai Wen},
url = {https://doi.org/10.48550/arXiv.2311.18451},
doi = {10.48550/ARXIV.2311.18451},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2311.18451},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2404.
Benmeziane, Hadjer; Ouarnoughi, Hamza; Niar, Smail; Maghraoui, Kaoutar El
Pareto Rank-Preserving Supernetwork for HW-NAS Technical Report
2023.
@techreport{<LineBreak>benmeziane2023pareto,
title = {Pareto Rank-Preserving Supernetwork for HW-NAS},
author = {Hadjer Benmeziane and Hamza Ouarnoughi and Smail Niar and Kaoutar El Maghraoui},
url = {https://www.researchgate.net/publication/374319863_Pareto_Rank-Preserving_Supernetwork_for_Hardware-Aware_Neural_Architecture_Search},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2403.
Mohammed, Mubarek
Model Compression Beyond Size Reduction Proceedings Article
In: Conference on Parsimony and Learning (Recent Spotlight Track), 2023.
@inproceedings{<LineBreak>mohammed2023model,
title = {Model Compression Beyond Size Reduction},
author = {Mubarek Mohammed},
url = {https://openreview.net/forum?id=HO0RdLgQtW},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {Conference on Parsimony and Learning (Recent Spotlight Track)},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
2402.
Wan, Qiyu; Wang, Lening; Wang, Jing; Song, Shuaiwen Leon; Fu, Xin
NAS-SE: Designing A Highly-Efficient In-Situ Neural Architecture Search Engine for Large-Scale Deployment Proceedings Article
In: Proceedings of the 56th Annual IEEE/ACM International Symposium on Microarchitecture, pp. 756–768, Association for Computing Machinery, <conf-loc>, <city>Toronto</city>, <state>ON</state>, <country>Canada</country>, </conf-loc>, 2023, ISBN: 9798400703294.
@inproceedings{10.1145/3613424.3614265,
title = {NAS-SE: Designing A Highly-Efficient In-Situ Neural Architecture Search Engine for Large-Scale Deployment},
author = {Qiyu Wan and Lening Wang and Jing Wang and Shuaiwen Leon Song and Xin Fu},
url = {https://doi.org/10.1145/3613424.3614265},
doi = {10.1145/3613424.3614265},
isbn = {9798400703294},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {Proceedings of the 56th Annual IEEE/ACM International Symposium on Microarchitecture},
pages = {756–768},
publisher = {Association for Computing Machinery},
address = {<conf-loc>, <city>Toronto</city>, <state>ON</state>, <country>Canada</country>, </conf-loc>},
series = {MICRO '23},
abstract = {The emergence of Neural Architecture Search (NAS) enables an automated neural network development process that potentially replaces manually-enabled machine learning expertise. A state-of-the-art NAS method, namely One-Shot NAS, has been proposed to drastically reduce the lengthy search time for a wide spectrum of conventional NAS methods. Nevertheless, the search cost is still prohibitively expensive for practical large-scale deployment with real-world applications. In this paper, we reveal that the fundamental cause for inefficient deployment of One-Shot NAS in both single-device and large-scale scenarios originates from the massive redundant off-chip weight access during the numerous DNN inference in sequential searching. Inspired by its algorithmic characteristics, we depart from the traditional CMOS-based architecture designs and propose a promising processing-in-memory design alternative to perform in-situ architecture search, which helps fundamentally address the redundancy issue. Moreover, we further discovered two major performance challenges of directly porting the searching process onto the existing PIM-based accelerators: severe pipeline contention and resource under-utilization. By leveraging these insights, we propose the first highly-efficient in-situ One-Shot NAS search engine design, named NAS-SE, for both single-device and large-scale deployment scenarios. NAS-SE is equipped with a two-phased network diversification strategy for eliminating resource contention, and a novel hardware mapping scheme for boosting the resource utilization by an order of magnitude. Our extensive evaluation demonstrates that NAS-SE significantly outperforms the state-of-the-art digital-based customized NAS accelerator (NASA) with an average speedup of 8.8 × and energy-efficiency improvement of 2.05 ×.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
The emergence of Neural Architecture Search (NAS) enables an automated neural network development process that potentially replaces manually-enabled machine learning expertise. A state-of-the-art NAS method, namely One-Shot NAS, has been proposed to drastically reduce the lengthy search time for a wide spectrum of conventional NAS methods. Nevertheless, the search cost is still prohibitively expensive for practical large-scale deployment with real-world applications. In this paper, we reveal that the fundamental cause for inefficient deployment of One-Shot NAS in both single-device and large-scale scenarios originates from the massive redundant off-chip weight access during the numerous DNN inference in sequential searching. Inspired by its algorithmic characteristics, we depart from the traditional CMOS-based architecture designs and propose a promising processing-in-memory design alternative to perform in-situ architecture search, which helps fundamentally address the redundancy issue. Moreover, we further discovered two major performance challenges of directly porting the searching process onto the existing PIM-based accelerators: severe pipeline contention and resource under-utilization. By leveraging these insights, we propose the first highly-efficient in-situ One-Shot NAS search engine design, named NAS-SE, for both single-device and large-scale deployment scenarios. NAS-SE is equipped with a two-phased network diversification strategy for eliminating resource contention, and a novel hardware mapping scheme for boosting the resource utilization by an order of magnitude. Our extensive evaluation demonstrates that NAS-SE significantly outperforms the state-of-the-art digital-based customized NAS accelerator (NASA) with an average speedup of 8.8 × and energy-efficiency improvement of 2.05 ×.
2401.
Ghebriout, Mohamed Imed Eddine; Bouzidi, Halima; Niar, Sma"ıl; Ouarnoughi, Hamza
Harmonic-NAS: Hardware-Aware Multimodal Neural Architecture Search on Resource-constrained Devices Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2309-06612b,
title = {Harmonic-NAS: Hardware-Aware Multimodal Neural Architecture Search on Resource-constrained Devices},
author = {Mohamed Imed Eddine Ghebriout and Halima Bouzidi and Sma"ıl Niar and Hamza Ouarnoughi},
url = {https://doi.org/10.48550/arXiv.2309.06612},
doi = {10.48550/ARXIV.2309.06612},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2309.06612},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2400.
McDermott, Luke; Weitz, Jason; Demler, Dmitri; Cummings, Daniel; Tran, Nhan; Duarte, Javier
Neural Architecture Codesign for Fast Bragg Peak Analysis Technical Report
2023.
@techreport{mcdermott2023neural,
title = {Neural Architecture Codesign for Fast Bragg Peak Analysis},
author = {Luke McDermott and Jason Weitz and Dmitri Demler and Daniel Cummings and Nhan Tran and Javier Duarte},
url = {https://arxiv.org/abs/2312.05978},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2399.
Lee, Jaeseong; hwang,
Multilingual Lottery Tickets to Pretrain Language Models Proceedings Article
In: The 2023 Conference on Empirical Methods in Natural Language Processing, 2023.
@inproceedings{<LineBreak>lee2023multilingual,
title = {Multilingual Lottery Tickets to Pretrain Language Models},
author = {Jaeseong Lee and hwang},
url = {https://openreview.net/forum?id=QG4BWnsX6m},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {The 2023 Conference on Empirical Methods in Natural Language Processing},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
2398.
Arabi, Pau Mulet; Flowers, Alec; Mauch, Lukas; Cardinaux, Fabien
DBsurf: A Discrepancy Based Method for Discrete Stochastic Gradient Estimation Technical Report
2023.
@techreport{arabi2023dbsurf,
title = {DBsurf: A Discrepancy Based Method for Discrete Stochastic Gradient Estimation},
author = {Pau Mulet Arabi and Alec Flowers and Lukas Mauch and Fabien Cardinaux},
url = {https://arxiv.org/abs/2309.03974},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2397.
Qin, Huafeng; Fan, Chao; Deng, Shaojiang; Li, Yantao; El-Yacoubi, Mounim A.; Zhou, Gang
AG-NAS: An Attention GRU-based Neural Architecture Search for Finger-Vein Recognition Journal Article
In: IEEE Transactions on Information Forensics and Security, pp. 1-1, 2023.
@article{10348535,
title = {AG-NAS: An Attention GRU-based Neural Architecture Search for Finger-Vein Recognition},
author = {Huafeng Qin and Chao Fan and Shaojiang Deng and Yantao Li and Mounim A. El-Yacoubi and Gang Zhou},
url = {https://ieeexplore.ieee.org/abstract/document/10348535},
doi = {10.1109/TIFS.2023.3340915},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {IEEE Transactions on Information Forensics and Security},
pages = {1-1},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
2396.
ZiWen, Dou; Dong, Ye
Multi-objective Neural Architecture Search for Efficient and Fast Semantic Segmentation on Edge Journal Article
In: IEEE Transactions on Intelligent Vehicles, pp. 1-12, 2023.
@article{10316624,
title = {Multi-objective Neural Architecture Search for Efficient and Fast Semantic Segmentation on Edge},
author = {Dou ZiWen and Ye Dong},
url = {https://ieeexplore.ieee.org/abstract/document/10316624},
doi = {10.1109/TIV.2023.3332594},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {IEEE Transactions on Intelligent Vehicles},
pages = {1-12},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
2395.
Yang, Changdi; Sheng, Yi; Dong, Peiyan; Kong, Zhenglun; Li, Yanyu; Yu, Pinrui; Yang, Lei; Lin, Xue; Wang, Yanzhi
Fast and Fair Medical AI on the Edge Through Neural Architecture Search for Hybrid Vision Models Proceedings Article
In: 2023 IEEE/ACM International Conference on Computer Aided Design (ICCAD), pp. 01-09, 2023.
@inproceedings{10323652,
title = {Fast and Fair Medical AI on the Edge Through Neural Architecture Search for Hybrid Vision Models},
author = {Changdi Yang and Yi Sheng and Peiyan Dong and Zhenglun Kong and Yanyu Li and Pinrui Yu and Lei Yang and Xue Lin and Yanzhi Wang},
doi = {10.1109/ICCAD57390.2023.10323652},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {2023 IEEE/ACM International Conference on Computer Aided Design (ICCAD)},
pages = {01-09},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
2394.
Poliakov, Egor; Hung, Wei-Jie; Huang, Ching-Chun
Efficient Constraint-Aware Neural Architecture Search for Object Detection Proceedings Article
In: 2023 Asia Pacific Signal and Information Processing Association Annual Summit and Conference (APSIPA ASC), pp. 733-740, 2023.
@inproceedings{10317340,
title = {Efficient Constraint-Aware Neural Architecture Search for Object Detection},
author = {Egor Poliakov and Wei-Jie Hung and Ching-Chun Huang},
doi = {10.1109/APSIPAASC58517.2023.10317340},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {2023 Asia Pacific Signal and Information Processing Association Annual Summit and Conference (APSIPA ASC)},
pages = {733-740},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
2393.
Liu, Lian; Wang, Ying; Zhao, Xiandong; Chen, Weiwei; Li, Huawei; Li, Xiaowei; Han, Yinhe
An Automatic Neural Network Architecture-and-Quantization Joint Optimization Framework for Efficient Model Inference Journal Article
In: IEEE Transactions on Computer-Aided Design of Integrated Circuits and Systems, pp. 1-1, 2023.
@article{10342823,
title = {An Automatic Neural Network Architecture-and-Quantization Joint Optimization Framework for Efficient Model Inference},
author = {Lian Liu and Ying Wang and Xiandong Zhao and Weiwei Chen and Huawei Li and Xiaowei Li and Yinhe Han},
url = {https://ieeexplore.ieee.org/abstract/document/10342823},
doi = {10.1109/TCAD.2023.3339438},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {IEEE Transactions on Computer-Aided Design of Integrated Circuits and Systems},
pages = {1-1},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
2392.
Sharifi, Zaniar; Soltanian, Khabat; Amiri, Ali
Developing Convolutional Neural Networks using a Novel Lamarckian Co-Evolutionary Algorithm Proceedings Article
In: 2023 13th International Conference on Computer and Knowledge Engineering (ICCKE), pp. 399-408, 2023.
@inproceedings{10326238,
title = {Developing Convolutional Neural Networks using a Novel Lamarckian Co-Evolutionary Algorithm},
author = {Zaniar Sharifi and Khabat Soltanian and Ali Amiri},
url = {https://ieeexplore.ieee.org/abstract/document/10326238},
doi = {10.1109/ICCKE60553.2023.10326238},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {2023 13th International Conference on Computer and Knowledge Engineering (ICCKE)},
pages = {399-408},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
2391.
Li, Xiangyi; Guo, Jiajia; Wen, Chao Kai; Tian, Wenqiang; Jin, Shi
Automatic Neural Network Design of Scene-customization for Massive MIMO CSI Feedback Proceedings Article
In: 2023 IEEE 98th Vehicular Technology Conference (VTC2023-Fall), pp. 1-5, 2023.
@inproceedings{10333588,
title = {Automatic Neural Network Design of Scene-customization for Massive MIMO CSI Feedback},
author = {Xiangyi Li and Jiajia Guo and Chao Kai Wen and Wenqiang Tian and Shi Jin},
doi = {10.1109/VTC2023-Fall60731.2023.10333588},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {2023 IEEE 98th Vehicular Technology Conference (VTC2023-Fall)},
pages = {1-5},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
2390.
Liu, Xuan; Liu, Xiong; Huang, Zhifeng; Zhao, Tianyang
A Compact Neural Network-Based Conversion Loss Model with Hard Constraints for Energy Management Journal Article
In: IEEE Transactions on Industry Applications, pp. 1-13, 2023.
@article{10323525,
title = {A Compact Neural Network-Based Conversion Loss Model with Hard Constraints for Energy Management},
author = {Xuan Liu and Xiong Liu and Zhifeng Huang and Tianyang Zhao},
doi = {10.1109/TIA.2023.3334698},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {IEEE Transactions on Industry Applications},
pages = {1-13},
keywords = {},
pubstate = {published},
tppubtype = {article}
}