Maintained by Difan Deng and Marius Lindauer.
The following list considers papers related to neural architecture search. It is by no means complete. If you miss a paper on the list, please let us know.
Please note that although NAS methods steadily improve, the quality of empirical evaluations in this field are still lagging behind compared to other areas in machine learning, AI and optimization. We would therefore like to share some best practices for empirical evaluations of NAS methods, which we believe will facilitate sustained and measurable progress in the field. If you are interested in a teaser, please read our blog post or directly jump to our checklist.
Transformers have gained increasing popularity in different domains. For a comprehensive list of papers focusing on Neural Architecture Search for Transformer-Based spaces, the awesome-transformer-search repo is all you need.
2023
2486.
Nair, Saeejith; Chen, Yuhao; Shafiee, Mohammad Javad; Wong, Alexander
NAS-NeRF: Generative Neural Architecture Search for Neural Radiance Fields Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2309-14293,
title = {NAS-NeRF: Generative Neural Architecture Search for Neural Radiance Fields},
author = {Saeejith Nair and Yuhao Chen and Mohammad Javad Shafiee and Alexander Wong},
url = {https://doi.org/10.48550/arXiv.2309.14293},
doi = {10.48550/ARXIV.2309.14293},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2309.14293},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2485.
Underwood, Robert; Madhyastha, Meghana; Burns, Randal C.; Nicolae, Bogdan
Understanding Patterns of Deep Learning ModelEvolution in Network Architecture Search Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2309-12576,
title = {Understanding Patterns of Deep Learning ModelEvolution in Network Architecture Search},
author = {Robert Underwood and Meghana Madhyastha and Randal C. Burns and Bogdan Nicolae},
url = {https://doi.org/10.48550/arXiv.2309.12576},
doi = {10.48550/ARXIV.2309.12576},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2309.12576},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2484.
Lyu, Kuangda; Li, Hao; Gong, Maoguo; Xing, Lining; Qin, A. K.
Surrogate-Assisted Evolutionary Multiobjective Neural Architecture Search based on Transfer Stacking and Knowledge Distillation Journal Article
In: IEEE Transactions on Evolutionary Computation, pp. 1-1, 2023.
@article{10263998,
title = {Surrogate-Assisted Evolutionary Multiobjective Neural Architecture Search based on Transfer Stacking and Knowledge Distillation},
author = {Kuangda Lyu and Hao Li and Maoguo Gong and Lining Xing and A. K. Qin},
url = {https://ieeexplore.ieee.org/abstract/document/10263998},
doi = {10.1109/TEVC.2023.3319567},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {IEEE Transactions on Evolutionary Computation},
pages = {1-1},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
2483.
Dolatabadi, Amirhossein; Chakravorty, Jhelum; Feng, Xiaoming
Supervised Federated Neural Architecture Search and Its Application in Power System Forecasting Proceedings Article
In: 2023 IEEE Power & Energy Society General Meeting (PESGM), pp. 1-5, 2023.
@inproceedings{10252675,
title = {Supervised Federated Neural Architecture Search and Its Application in Power System Forecasting},
author = {Amirhossein Dolatabadi and Jhelum Chakravorty and Xiaoming Feng},
doi = {10.1109/PESGM52003.2023.10252675},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {2023 IEEE Power & Energy Society General Meeting (PESGM)},
pages = {1-5},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
2482.
Priyadarshi, Sweta; Jiang, Tianyu; Cheng, Hsin-Pai; Krishna, Sendil; Ganapathy, Viswanath; Patel, Chirag
DONNAv2 - Lightweight Neural Architecture Search for Vision tasks Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2309-14670,
title = {DONNAv2 - Lightweight Neural Architecture Search for Vision tasks},
author = {Sweta Priyadarshi and Tianyu Jiang and Hsin-Pai Cheng and Sendil Krishna and Viswanath Ganapathy and Chirag Patel},
url = {https://doi.org/10.48550/arXiv.2309.14670},
doi = {10.48550/ARXIV.2309.14670},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2309.14670},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2481.
Li, Yangyang; Liu, Ruijiao; Hao, Xiaobin; Shang, Ronghua; Zhao, Peixiang; Jiao, Licheng
EQNAS: Evolutionary Quantum Neural Architecture Search for Image Classification Journal Article
In: Neural Networks, vol. 168, pp. 471-483, 2023, ISSN: 0893-6080.
@article{LI2023471,
title = {EQNAS: Evolutionary Quantum Neural Architecture Search for Image Classification},
author = {Yangyang Li and Ruijiao Liu and Xiaobin Hao and Ronghua Shang and Peixiang Zhao and Licheng Jiao},
url = {https://www.sciencedirect.com/science/article/pii/S0893608023005348},
doi = {https://doi.org/10.1016/j.neunet.2023.09.040},
issn = {0893-6080},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {Neural Networks},
volume = {168},
pages = {471-483},
abstract = {Quantum neural network (QNN) is a neural network model based on the principles of quantum mechanics. The advantages of faster computing speed, higher memory capacity, smaller network size and elimination of catastrophic amnesia make it a new idea to solve the problem of training massive data that is difficult for classical neural networks. However, the quantum circuit of QNN are artificially designed with high circuit complexity and low precision in classification tasks. In this paper, a neural architecture search method EQNAS is proposed to improve QNN. First, initializing the quantum population after image quantum encoding. The next step is observing the quantum population and evaluating the fitness. The last is updating the quantum population. Quantum rotation gate update, quantum circuit construction and entirety interference crossover are specific operations. The last two steps need to be carried out iteratively until a satisfactory fitness is achieved. After a lot of experiments on the searched quantum neural networks, the feasibility and effectiveness of the algorithm proposed in this paper are proved, and the searched QNN is obviously better than the original algorithm. The classification accuracy on the mnist dataset and the warship dataset not only increased by 5.31% and 4.52%, respectively, but also reduced the parameters by 21.88% and 31.25% respectively. Code will be available at https://gitee.com/Pcyslist/models/tree/master/research/cv/EQNAS, and https://github.com/Pcyslist/EQNAS.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Quantum neural network (QNN) is a neural network model based on the principles of quantum mechanics. The advantages of faster computing speed, higher memory capacity, smaller network size and elimination of catastrophic amnesia make it a new idea to solve the problem of training massive data that is difficult for classical neural networks. However, the quantum circuit of QNN are artificially designed with high circuit complexity and low precision in classification tasks. In this paper, a neural architecture search method EQNAS is proposed to improve QNN. First, initializing the quantum population after image quantum encoding. The next step is observing the quantum population and evaluating the fitness. The last is updating the quantum population. Quantum rotation gate update, quantum circuit construction and entirety interference crossover are specific operations. The last two steps need to be carried out iteratively until a satisfactory fitness is achieved. After a lot of experiments on the searched quantum neural networks, the feasibility and effectiveness of the algorithm proposed in this paper are proved, and the searched QNN is obviously better than the original algorithm. The classification accuracy on the mnist dataset and the warship dataset not only increased by 5.31% and 4.52%, respectively, but also reduced the parameters by 21.88% and 31.25% respectively. Code will be available at https://gitee.com/Pcyslist/models/tree/master/research/cv/EQNAS, and https://github.com/Pcyslist/EQNAS.
2480.
Jin, Xiu; Xiong, Jianghui; Rao, Yuan; Zhang, Tong; Ba, Wenjing; Gu, Shangfeng; Zhang, Xiaodan; Lu, Jie
TranNas-NirCR: A method for improving the diagnosis of asymptomatic wheat scab with transfer learning and neural architecture search Journal Article
In: Computers and Electronics in Agriculture, vol. 213, pp. 108271, 2023, ISSN: 0168-1699.
@article{JIN2023108271,
title = {TranNas-NirCR: A method for improving the diagnosis of asymptomatic wheat scab with transfer learning and neural architecture search},
author = {Xiu Jin and Jianghui Xiong and Yuan Rao and Tong Zhang and Wenjing Ba and Shangfeng Gu and Xiaodan Zhang and Jie Lu},
url = {https://www.sciencedirect.com/science/article/pii/S0168169923006592},
doi = {https://doi.org/10.1016/j.compag.2023.108271},
issn = {0168-1699},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {Computers and Electronics in Agriculture},
volume = {213},
pages = {108271},
abstract = {Wheat scab is one of the most important diseases endangering the health of wheat which severely affects the yield and quality of wheat. Thus, the diagnosis of wheat scab is very important. However, it is difficult to distinguish between asymptomatic wheats without visible spots on the surface and healthy wheats by traditional visual methods under natural conditions, which has greatly hindered the diagnosis of wheat scab. To address the challenge of poor model classification caused by the difficulty in distinguishing asymptomatic wheats from healthy wheats, we use near-infrared spectral data with healthy, symptomatic and indistinguishable asymptomatic wheats and propose a new approach Transfer Learning and Neural Architecture Search for Near-infrared with Convolutional Networks and Recurrent Networks (TranNas-NirCR). This approach integrates neural architecture search with transfer learning and employs a combination of convolutional neural networks and recurrent neural networks in the search space. Compared to other methods, the TranNas-NirCR method achieved better classification results with an accuracy of 90.42%, which is 2.68% higher than support vector machines (SVM), 5.36% higher than neural architecture search (NAS), and 4.21% higher than Transfer Learning with Neural Architecture Search (Tran_NAS). This method shows strong generalization performance in the case of only a small amount of near-infrared spectral data, which is of referential significance for diagnosing early wheat scab in real conditions.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Wheat scab is one of the most important diseases endangering the health of wheat which severely affects the yield and quality of wheat. Thus, the diagnosis of wheat scab is very important. However, it is difficult to distinguish between asymptomatic wheats without visible spots on the surface and healthy wheats by traditional visual methods under natural conditions, which has greatly hindered the diagnosis of wheat scab. To address the challenge of poor model classification caused by the difficulty in distinguishing asymptomatic wheats from healthy wheats, we use near-infrared spectral data with healthy, symptomatic and indistinguishable asymptomatic wheats and propose a new approach Transfer Learning and Neural Architecture Search for Near-infrared with Convolutional Networks and Recurrent Networks (TranNas-NirCR). This approach integrates neural architecture search with transfer learning and employs a combination of convolutional neural networks and recurrent neural networks in the search space. Compared to other methods, the TranNas-NirCR method achieved better classification results with an accuracy of 90.42%, which is 2.68% higher than support vector machines (SVM), 5.36% higher than neural architecture search (NAS), and 4.21% higher than Transfer Learning with Neural Architecture Search (Tran_NAS). This method shows strong generalization performance in the case of only a small amount of near-infrared spectral data, which is of referential significance for diagnosing early wheat scab in real conditions.
2479.
Chitty-Venkata, Krishna Teja; Bian, Yiming; Emani, Murali; Vishwanath, Venkatram; Somani, Arun K.
Differentiable Neural Architecture, Mixed Precision and Accelerator Co-Search Technical Report
2023.
@techreport{DBLP:journals/access/ChittyVenkataBEVS23,
title = {Differentiable Neural Architecture, Mixed Precision and Accelerator Co-Search},
author = {Krishna Teja Chitty-Venkata and Yiming Bian and Murali Emani and Venkatram Vishwanath and Arun K. Somani},
url = {https://doi.org/10.1109/ACCESS.2023.3320133},
doi = {10.1109/ACCESS.2023.3320133},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {IEEE Access},
volume = {11},
pages = {106670–106687},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2478.
Yang, Dong; Xu, Ziyue; He, Yufan; Nath, Vishwesh; Li, Wenqi; Myronenko, Andriy; Hatamizadeh, Ali; Zhao, Can; Roth, Holger R.; Xu, Daguang
DAST: Differentiable Architecture Search with Transformer for 3D Medical Image Segmentation Proceedings Article
In: Greenspan, Hayit; Madabhushi, Anant; Mousavi, Parvin; Salcudean, Septimiu; Duncan, James; Syeda-Mahmood, Tanveer; Taylor, Russell (Ed.): Medical Image Computing and Computer Assisted Intervention – MICCAI 2023, pp. 747–756, Springer Nature Switzerland, Cham, 2023, ISBN: 978-3-031-43898-1.
@inproceedings{10.1007/978-3-031-43898-1_71,
title = {DAST: Differentiable Architecture Search with Transformer for 3D Medical Image Segmentation},
author = {Dong Yang and Ziyue Xu and Yufan He and Vishwesh Nath and Wenqi Li and Andriy Myronenko and Ali Hatamizadeh and Can Zhao and Holger R. Roth and Daguang Xu},
editor = {Hayit Greenspan and Anant Madabhushi and Parvin Mousavi and Septimiu Salcudean and James Duncan and Tanveer Syeda-Mahmood and Russell Taylor},
url = {https://link.springer.com/chapter/10.1007/978-3-031-43898-1_71},
isbn = {978-3-031-43898-1},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {Medical Image Computing and Computer Assisted Intervention – MICCAI 2023},
pages = {747–756},
publisher = {Springer Nature Switzerland},
address = {Cham},
abstract = {Neural Architecture Search (NAS) has been widely used for medical image segmentation by improving both model performance and computational efficiency. Recently, the Visual Transformer (ViT) model has achieved significant success in computer vision tasks. Leveraging these two innovations, we propose a novel NAS algorithm, DAST, to optimize neural network models with transformers for 3D medical image segmentation. The proposed algorithm is able to search the global structure and local operations of the architecture with a GPU memory consumption constraint. The resulting architectures reveal an effective relationship between convolution and transformer layers in segmentation models. Moreover, we validate the proposed algorithm on large-scale medical image segmentation data sets, showing its superior performance over the baselines. The model achieves state-of-the-art performance in the public challenge of kidney CT segmentation (KiTS'19).},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Neural Architecture Search (NAS) has been widely used for medical image segmentation by improving both model performance and computational efficiency. Recently, the Visual Transformer (ViT) model has achieved significant success in computer vision tasks. Leveraging these two innovations, we propose a novel NAS algorithm, DAST, to optimize neural network models with transformers for 3D medical image segmentation. The proposed algorithm is able to search the global structure and local operations of the architecture with a GPU memory consumption constraint. The resulting architectures reveal an effective relationship between convolution and transformer layers in segmentation models. Moreover, we validate the proposed algorithm on large-scale medical image segmentation data sets, showing its superior performance over the baselines. The model achieves state-of-the-art performance in the public challenge of kidney CT segmentation (KiTS'19).
2477.
Xu, Hongyan; Wang, Dadong; Sowmya, Arcot; Katz, Ian
Detection of Basal Cell Carcinoma in Whole Slide Images Proceedings Article
In: Greenspan, Hayit; Madabhushi, Anant; Mousavi, Parvin; Salcudean, Septimiu; Duncan, James; Syeda-Mahmood, Tanveer; Taylor, Russell (Ed.): Medical Image Computing and Computer Assisted Intervention – MICCAI 2023, pp. 263–272, Springer Nature Switzerland, Cham, 2023, ISBN: 978-3-031-43987-2.
@inproceedings{10.1007/978-3-031-43987-2_26,
title = {Detection of Basal Cell Carcinoma in Whole Slide Images},
author = {Hongyan Xu and Dadong Wang and Arcot Sowmya and Ian Katz},
editor = {Hayit Greenspan and Anant Madabhushi and Parvin Mousavi and Septimiu Salcudean and James Duncan and Tanveer Syeda-Mahmood and Russell Taylor},
url = {https://link.springer.com/chapter/10.1007/978-3-031-43987-2_26},
isbn = {978-3-031-43987-2},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {Medical Image Computing and Computer Assisted Intervention – MICCAI 2023},
pages = {263–272},
publisher = {Springer Nature Switzerland},
address = {Cham},
abstract = {Basal cell carcinoma (BCC) is a prevalent and increasingly diagnosed form of skin cancer that can benefit from automated whole slide image (WSI) analysis. However, traditional methods that utilize popular network structures designed for natural images, such as the ImageNet dataset, may result in reduced accuracy due to the significant differences between natural and pathology images. In this paper, we analyze skin cancer images using the optimal network obtained by neural architecture search (NAS) on the skin cancer dataset. Compared with traditional methods, our network is more applicable to the task of skin cancer detection. Furthermore, unlike traditional unilaterally augmented (UA) methods, the proposed supernet Skin-Cancer net (SC-net) considers the fairness of training and alleviates the effects of evaluation bias. We use the SC-net to fairly treat all the architectures in the search space and leveraged evolutionary search to obtain the optimal architecture for a skin cancer dataset. Our experiments involve 277,000 patches split from 194 slides. Under the same FLOPs budget (4.1G), our searched ResNet50 model achieves 96.2% accuracy and 96.5% area under the ROC curve (AUC), which are 4.8% and 4.7% higher than those with the baseline settings, respectively.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Basal cell carcinoma (BCC) is a prevalent and increasingly diagnosed form of skin cancer that can benefit from automated whole slide image (WSI) analysis. However, traditional methods that utilize popular network structures designed for natural images, such as the ImageNet dataset, may result in reduced accuracy due to the significant differences between natural and pathology images. In this paper, we analyze skin cancer images using the optimal network obtained by neural architecture search (NAS) on the skin cancer dataset. Compared with traditional methods, our network is more applicable to the task of skin cancer detection. Furthermore, unlike traditional unilaterally augmented (UA) methods, the proposed supernet Skin-Cancer net (SC-net) considers the fairness of training and alleviates the effects of evaluation bias. We use the SC-net to fairly treat all the architectures in the search space and leveraged evolutionary search to obtain the optimal architecture for a skin cancer dataset. Our experiments involve 277,000 patches split from 194 slides. Under the same FLOPs budget (4.1G), our searched ResNet50 model achieves 96.2% accuracy and 96.5% area under the ROC curve (AUC), which are 4.8% and 4.7% higher than those with the baseline settings, respectively.
2476.
Ghosh, Arjun; Jana, Nanda Dulal; Das, Swagatam; Mallipeddi, Rammohan
Two-Phase Evolutionary Convolutional Neural Network Architecture Search for Medical Image Classification Journal Article
In: IEEE Access, vol. 11, pp. 115280-115305, 2023.
@article{10278411,
title = {Two-Phase Evolutionary Convolutional Neural Network Architecture Search for Medical Image Classification},
author = {Arjun Ghosh and Nanda Dulal Jana and Swagatam Das and Rammohan Mallipeddi},
url = {https://ieeexplore.ieee.org/document/10278411},
doi = {10.1109/ACCESS.2023.3323705},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {IEEE Access},
volume = {11},
pages = {115280-115305},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
2475.
Wang, Hao; Han, Dezhi; Cui, Mingming; Chen, Chongqing
NAS-YOLOX: a SAR ship detection using neural architecture search and multi-scale attention Journal Article
In: Connect. Sci., vol. 35, no. 1, pp. 1–32, 2023.
@article{DBLP:journals/connection/WangHCC23,
title = {NAS-YOLOX: a SAR ship detection using neural architecture search and multi-scale attention},
author = {Hao Wang and Dezhi Han and Mingming Cui and Chongqing Chen},
url = {https://doi.org/10.1080/09540091.2023.2257399},
doi = {10.1080/09540091.2023.2257399},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {Connect. Sci.},
volume = {35},
number = {1},
pages = {1–32},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
2474.
Wang, Haishuai; Gao, Yang; Zheng, Xin; Zhang, Peng; Chen, Hongyang; Bu, Jiajun
Graph Neural Architecture Search with GPT-4 Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2310-01436,
title = {Graph Neural Architecture Search with GPT-4},
author = {Haishuai Wang and Yang Gao and Xin Zheng and Peng Zhang and Hongyang Chen and Jiajun Bu},
url = {https://doi.org/10.48550/arXiv.2310.01436},
doi = {10.48550/ARXIV.2310.01436},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2310.01436},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2473.
Yang, Shangshang; Yu, Xiaoshan; Tian, Ye; Yan, Xueming; Ma, Haiping; Zhang, Xingyi
Evolutionary Neural Architecture Search for Transformer in Knowledge Tracing Journal Article
In: NeurIPS 2023, vol. abs/2310.01180, 2023.
@article{DBLP:journals/corr/abs-2310-01180,
title = {Evolutionary Neural Architecture Search for Transformer in Knowledge Tracing},
author = {Shangshang Yang and Xiaoshan Yu and Ye Tian and Xueming Yan and Haiping Ma and Xingyi Zhang},
url = {https://doi.org/10.48550/arXiv.2310.01180},
doi = {10.48550/ARXIV.2310.01180},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = { NeurIPS 2023},
volume = {abs/2310.01180},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
2472.
A First Look at Generating Website Fingerprinting Attacks via Neural Architecture Search Proceedings Article
In: WPES ’23, UWSpace, 2023.
@inproceedings{Limam,
title = {A First Look at Generating Website Fingerprinting Attacks via Neural Architecture Search},
url = {http://hdl.handle.net/10012/20020},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {WPES ’23},
publisher = {UWSpace},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
2471.
Gao, Jiahui
Language models in NLP : from architecture design to downstream application Bachelor Thesis
2023.
@bachelorthesis{HKUHUB_10722_332194,
title = {Language models in NLP : from architecture design to downstream application},
author = {Jiahui Gao},
url = {http://hdl.handle.net/10722/332194},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {HKU Theses Online (HKUTO)},
abstract = {Natural language processing (NLP) is a rapidly evolving field, and language models (LMs) play a critical role in advancing research in various NLP tasks, such as language generation, machine translation, sentiment analysis, and question-answering. This thesis presents our contributions toward advancing the research in language models from two perspectives: the design of language model architecture and downstream application.
In the first part of the thesis, we aim to enhance the ability of pre-trained language models by discovering an efficient and powerful architecture. Instead of resorting to manual design, we pioneer an approach to automatically discover novel pre-trained language model (PLM) backbone within a flexible search space. To this end, we introduce an efficient Neural Architecture Search (NAS) method, termed OP-NAS, which concurrently optimizes the search algorithm and the evaluation of prospective models. The architecture discovered through this process, referred to as AutoBERT-Zero, significantly surpasses the performance of BERT and its variants across various downstream tasks, while also exhibiting exceptional transfer and scaling abilities.
In the second part of this thesis, we explore the practical applications of language models, drawing upon their recent success in the field. Specifically, we examine two primary directions: effective downstream adaptation and the extension of language models to broader domains beyond natural language processing (NLP). In particular, we first introduce SunGen, a novel framework that enables the efficient adaptation of pre-trained language models (PLMs) to downstream tasks. SunGen enhances the quality of PLM-generated data, allowing for the training of a compact task-specific model with substantially fewer parameters. This approach not only achieves superior performance to that of the original PLM but also offers greater efficiency during training and inference. Then, we demonstrate the potential of language models beyond NLP by presenting a novel unpaired cross-lingual method for generating image captions. This method enables captioning tasks to be performed for languages without any caption annotations, effectively bridging the gap between vision and language understanding across different languages. Overall, this thesis contributes to realizing the full potential of language models and provides new insights for future research in this rapidly evolving field.},
keywords = {},
pubstate = {published},
tppubtype = {bachelorthesis}
}
Natural language processing (NLP) is a rapidly evolving field, and language models (LMs) play a critical role in advancing research in various NLP tasks, such as language generation, machine translation, sentiment analysis, and question-answering. This thesis presents our contributions toward advancing the research in language models from two perspectives: the design of language model architecture and downstream application.
In the first part of the thesis, we aim to enhance the ability of pre-trained language models by discovering an efficient and powerful architecture. Instead of resorting to manual design, we pioneer an approach to automatically discover novel pre-trained language model (PLM) backbone within a flexible search space. To this end, we introduce an efficient Neural Architecture Search (NAS) method, termed OP-NAS, which concurrently optimizes the search algorithm and the evaluation of prospective models. The architecture discovered through this process, referred to as AutoBERT-Zero, significantly surpasses the performance of BERT and its variants across various downstream tasks, while also exhibiting exceptional transfer and scaling abilities.
In the second part of this thesis, we explore the practical applications of language models, drawing upon their recent success in the field. Specifically, we examine two primary directions: effective downstream adaptation and the extension of language models to broader domains beyond natural language processing (NLP). In particular, we first introduce SunGen, a novel framework that enables the efficient adaptation of pre-trained language models (PLMs) to downstream tasks. SunGen enhances the quality of PLM-generated data, allowing for the training of a compact task-specific model with substantially fewer parameters. This approach not only achieves superior performance to that of the original PLM but also offers greater efficiency during training and inference. Then, we demonstrate the potential of language models beyond NLP by presenting a novel unpaired cross-lingual method for generating image captions. This method enables captioning tasks to be performed for languages without any caption annotations, effectively bridging the gap between vision and language understanding across different languages. Overall, this thesis contributes to realizing the full potential of language models and provides new insights for future research in this rapidly evolving field.
In the first part of the thesis, we aim to enhance the ability of pre-trained language models by discovering an efficient and powerful architecture. Instead of resorting to manual design, we pioneer an approach to automatically discover novel pre-trained language model (PLM) backbone within a flexible search space. To this end, we introduce an efficient Neural Architecture Search (NAS) method, termed OP-NAS, which concurrently optimizes the search algorithm and the evaluation of prospective models. The architecture discovered through this process, referred to as AutoBERT-Zero, significantly surpasses the performance of BERT and its variants across various downstream tasks, while also exhibiting exceptional transfer and scaling abilities.
In the second part of this thesis, we explore the practical applications of language models, drawing upon their recent success in the field. Specifically, we examine two primary directions: effective downstream adaptation and the extension of language models to broader domains beyond natural language processing (NLP). In particular, we first introduce SunGen, a novel framework that enables the efficient adaptation of pre-trained language models (PLMs) to downstream tasks. SunGen enhances the quality of PLM-generated data, allowing for the training of a compact task-specific model with substantially fewer parameters. This approach not only achieves superior performance to that of the original PLM but also offers greater efficiency during training and inference. Then, we demonstrate the potential of language models beyond NLP by presenting a novel unpaired cross-lingual method for generating image captions. This method enables captioning tasks to be performed for languages without any caption annotations, effectively bridging the gap between vision and language understanding across different languages. Overall, this thesis contributes to realizing the full potential of language models and provides new insights for future research in this rapidly evolving field.
2470.
Wu, Yanling; Tang, Baoping; Deng, Lei; Shen, Yizhe
Hardware-Resource-Constrained Neural Architecture Search for Edge-Side Fault Diagnosis of Wind-Turbine Gearboxes Journal Article
In: IEEE Transactions on Industrial Electronics, pp. 1-11, 2023.
@article{10273707,
title = {Hardware-Resource-Constrained Neural Architecture Search for Edge-Side Fault Diagnosis of Wind-Turbine Gearboxes},
author = {Yanling Wu and Baoping Tang and Lei Deng and Yizhe Shen},
doi = {10.1109/TIE.2023.3317847},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {IEEE Transactions on Industrial Electronics},
pages = {1-11},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
2469.
Teixeira, Rafael; Antunes, Mário; Sobral, Rúben; Martins, João; Gomes, Diogo; Aguiar, Rui L.
Exploring the Intricacies of Neural Network Optimization Proceedings Article
In: Bifet, Albert; Lorena, Ana Carolina; Ribeiro, Rita P.; Gama, João; Abreu, Pedro H. (Ed.): Discovery Science, pp. 18–32, Springer Nature Switzerland, Cham, 2023, ISBN: 978-3-031-45275-8.
@inproceedings{10.1007/978-3-031-45275-8_2,
title = {Exploring the Intricacies of Neural Network Optimization},
author = {Rafael Teixeira and Mário Antunes and Rúben Sobral and João Martins and Diogo Gomes and Rui L. Aguiar},
editor = {Albert Bifet and Ana Carolina Lorena and Rita P. Ribeiro and João Gama and Pedro H. Abreu},
url = {https://link.springer.com/chapter/10.1007/978-3-031-45275-8_2},
isbn = {978-3-031-45275-8},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {Discovery Science},
pages = {18–32},
publisher = {Springer Nature Switzerland},
address = {Cham},
abstract = {Recent machine learning breakthroughs in computer vision and natural language processing were possible due to Deep Neural Networks (DNNs) learning capabilities. Even so, applying DNNs is quite challenging, as they usually have more hyperparameters than shallow models. The higher number of hyperparameters leads to allocating more time for model optimization and training to achieve optimal results. However, if there is a better understanding of the impact of each hyperparameter on the model performance, then one can decide which hyperparameters to optimize according to the available optimization budget or desired performance. This work analyzes the impact of the different hyperparameters when applying dense DNNs to tabular datasets. This is achieved by optimizing each hyperparameter individually and comparing their influence on the model performance. The results show that the batch size usually only affects training time, reducing it by up to 80% or increasing it by 200%. In contrast, the hidden layer size does not consistently affect the considered performance metrics. The optimizer can significantly affect the model's overall performance while also varying the training time, with Adam being the generally the better optimizer. Overall, we show that the hyperparameters do not equally affect the DNN and that some can be discarded if there is a constrained search budget.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Recent machine learning breakthroughs in computer vision and natural language processing were possible due to Deep Neural Networks (DNNs) learning capabilities. Even so, applying DNNs is quite challenging, as they usually have more hyperparameters than shallow models. The higher number of hyperparameters leads to allocating more time for model optimization and training to achieve optimal results. However, if there is a better understanding of the impact of each hyperparameter on the model performance, then one can decide which hyperparameters to optimize according to the available optimization budget or desired performance. This work analyzes the impact of the different hyperparameters when applying dense DNNs to tabular datasets. This is achieved by optimizing each hyperparameter individually and comparing their influence on the model performance. The results show that the batch size usually only affects training time, reducing it by up to 80% or increasing it by 200%. In contrast, the hidden layer size does not consistently affect the considered performance metrics. The optimizer can significantly affect the model's overall performance while also varying the training time, with Adam being the generally the better optimizer. Overall, we show that the hyperparameters do not equally affect the DNN and that some can be discarded if there is a constrained search budget.
2468.
Harb, Ala’a; Gad, Abdalla; Yaghi, Maha; Alhalabi, Marah; Zia, Huma; Yousaf, Jawad; Khelifi, Adel; Ghoudi, Kilani; Ghazal, Mohammed
Diverse distant-students deep emotion recognition and visualization Journal Article
In: Computers and Electrical Engineering, vol. 111, pp. 108963, 2023, ISSN: 0045-7906.
@article{HARB2023108963,
title = {Diverse distant-students deep emotion recognition and visualization},
author = {Ala’a Harb and Abdalla Gad and Maha Yaghi and Marah Alhalabi and Huma Zia and Jawad Yousaf and Adel Khelifi and Kilani Ghoudi and Mohammed Ghazal},
url = {https://www.sciencedirect.com/science/article/pii/S0045790623003877},
doi = {https://doi.org/10.1016/j.compeleceng.2023.108963},
issn = {0045-7906},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {Computers and Electrical Engineering},
volume = {111},
pages = {108963},
abstract = {Distance learning through online platforms has recently emerged as a significant addition to the educational system, providing support and enhancing traditional delivery modes in schools. Instructors need visual feedback to monitor students’ engagement. This research aims to investigate the emotional state of culturally diverse students in the country throughout the lecture and provide teachers with an interactive dashboard to monitor students’ emotional states during a particular lecture. We collected a specialized dataset that includes Arab students posing for various emotions specifically related to lecture scenarios. Face detection and alignment algorithm is then applied. Finally, we utilize Neural Architecture Search (NAS) to optimize emotion classification architecture. The overall accuracy of the model is 86.29% on our collected dataset and 98.84% on the CK+ dataset. The live testing of the system shows the emotions clearly with real-time feedback. The AI-powered dashboard gives instructors insights into student engagement during distance learning, which induces data-driven decisions to optimize teaching strategies.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Distance learning through online platforms has recently emerged as a significant addition to the educational system, providing support and enhancing traditional delivery modes in schools. Instructors need visual feedback to monitor students’ engagement. This research aims to investigate the emotional state of culturally diverse students in the country throughout the lecture and provide teachers with an interactive dashboard to monitor students’ emotional states during a particular lecture. We collected a specialized dataset that includes Arab students posing for various emotions specifically related to lecture scenarios. Face detection and alignment algorithm is then applied. Finally, we utilize Neural Architecture Search (NAS) to optimize emotion classification architecture. The overall accuracy of the model is 86.29% on our collected dataset and 98.84% on the CK+ dataset. The live testing of the system shows the emotions clearly with real-time feedback. The AI-powered dashboard gives instructors insights into student engagement during distance learning, which induces data-driven decisions to optimize teaching strategies.
2467.
Cordeiro, João Rala; Mosca, Sara; Correia-Costa, Ana; Ferreira, Cátia; Pimenta, Joana; Correia-Costa, Liane; Barros, Henrique; Postolache, Octavian
The Association between Childhood Obesity and Cardiovascular Changes in 10 Years Using Special Data Science Analysis Journal Article
In: Children, vol. 10, no. 10, 2023, ISSN: 2227-9067.
@article{children10101655,
title = {The Association between Childhood Obesity and Cardiovascular Changes in 10 Years Using Special Data Science Analysis},
author = {João Rala Cordeiro and Sara Mosca and Ana Correia-Costa and Cátia Ferreira and Joana Pimenta and Liane Correia-Costa and Henrique Barros and Octavian Postolache},
url = {https://www.mdpi.com/2227-9067/10/10/1655},
doi = {10.3390/children10101655},
issn = {2227-9067},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {Children},
volume = {10},
number = {10},
abstract = {The increasing prevalence of overweight and obesity is a worldwide problem, with several well-known consequences that might start to develop early in life during childhood. The present research based on data from children that have been followed since birth in a previously established cohort study (Generation XXI, Porto, Portugal), taking advantage of State-of-the-Art (SoA) data science techniques and methods, including Neural Architecture Search (NAS), explainable Artificial Intelligence (XAI), and Deep Learning (DL), aimed to explore the hidden value of data, namely on electrocardiogram (ECG) records performed during follow-up visits. The combination of these techniques allowed us to clarify subtle cardiovascular changes already present at 10 years of age, which are evident from ECG analysis and probably induced by the presence of obesity. The proposed novel combination of new methodologies and techniques is discussed, as well as their applicability in other health domains.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
The increasing prevalence of overweight and obesity is a worldwide problem, with several well-known consequences that might start to develop early in life during childhood. The present research based on data from children that have been followed since birth in a previously established cohort study (Generation XXI, Porto, Portugal), taking advantage of State-of-the-Art (SoA) data science techniques and methods, including Neural Architecture Search (NAS), explainable Artificial Intelligence (XAI), and Deep Learning (DL), aimed to explore the hidden value of data, namely on electrocardiogram (ECG) records performed during follow-up visits. The combination of these techniques allowed us to clarify subtle cardiovascular changes already present at 10 years of age, which are evident from ECG analysis and probably induced by the presence of obesity. The proposed novel combination of new methodologies and techniques is discussed, as well as their applicability in other health domains.
2466.
Nistor, Sergiu Cosmin; Jaradat, Mohammad; Nistor, Razvan Liviu
Developing an Algorithm for Fast Performance Estimation of Recurrent Memory Cells Journal Article
In: IEEE Access, vol. 11, pp. 112877–112890, 2023.
@article{DBLP:journals/access/NistorJN23,
title = {Developing an Algorithm for Fast Performance Estimation of Recurrent Memory Cells},
author = {Sergiu Cosmin Nistor and Mohammad Jaradat and Razvan Liviu Nistor},
url = {https://doi.org/10.1109/ACCESS.2023.3322367},
doi = {10.1109/ACCESS.2023.3322367},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {IEEE Access},
volume = {11},
pages = {112877–112890},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
2465.
Mao, Pengli; Lin, Yan; Zhang, Baochang; Li, Lin
UCB-Based Architecture Search for Remaining Useful Life Prediction Proceedings Article
In: Jia, Yingmin; Zhang, Weicun; Fu, Yongling; Wang, Jiqiang (Ed.): Proceedings of 2023 Chinese Intelligent Systems Conference, pp. 677–686, Springer Nature Singapore, Singapore, 2023, ISBN: 978-981-99-6886-2.
@inproceedings{10.1007/978-981-99-6886-2_58,
title = {UCB-Based Architecture Search for Remaining Useful Life Prediction},
author = {Pengli Mao and Yan Lin and Baochang Zhang and Lin Li},
editor = {Yingmin Jia and Weicun Zhang and Yongling Fu and Jiqiang Wang},
isbn = {978-981-99-6886-2},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {Proceedings of 2023 Chinese Intelligent Systems Conference},
pages = {677–686},
publisher = {Springer Nature Singapore},
address = {Singapore},
abstract = {Remaining useful life (RUL) is a significant challenge in prognostics and health management. With the development of data science and deep learning, neural network technology has been applied to this problem. In this study, we present a self-optimized mechanism for neural architecture search. Based on upper confidence bound (UCB) algorithm, we construct a reinforcement learning method for search strategy. UCB explores the combinatorial parameter space of a multi-head convolutional neural network with a recurrent neural network to find a suitable architecture. We conducted experiments on the C-MAPSS dataset, and our proposed model achieved better results in measured by RMSE and Scoring function. Compared with other data-based approaches, our method had better performance in terms of both precision and efficiency.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Remaining useful life (RUL) is a significant challenge in prognostics and health management. With the development of data science and deep learning, neural network technology has been applied to this problem. In this study, we present a self-optimized mechanism for neural architecture search. Based on upper confidence bound (UCB) algorithm, we construct a reinforcement learning method for search strategy. UCB explores the combinatorial parameter space of a multi-head convolutional neural network with a recurrent neural network to find a suitable architecture. We conducted experiments on the C-MAPSS dataset, and our proposed model achieved better results in measured by RMSE and Scoring function. Compared with other data-based approaches, our method had better performance in terms of both precision and efficiency.
2464.
Li, Wenhao; Su, Xiu; You, Shan; Wang, Fei; Qian, Chen; Xu, Chang
DiffNAS: Bootstrapping Diffusion Models by Prompting for Better Architectures Booklet
2023.
@booklet{DBLP:journals/corr/abs-2310-04750,
title = {DiffNAS: Bootstrapping Diffusion Models by Prompting for Better Architectures},
author = {Wenhao Li and Xiu Su and Shan You and Fei Wang and Chen Qian and Chang Xu},
url = {https://doi.org/10.48550/arXiv.2310.04750},
doi = {10.48550/ARXIV.2310.04750},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2310.04750},
month = {01},
keywords = {},
pubstate = {published},
tppubtype = {booklet}
}
2463.
Arhipov, Andrey; Fomin, Ivan
Multi-Objective Fine-Grained Neural Architecture Search Proceedings Article
In: 2023 International Russian Automation Conference (RusAutoCon), pp. 164-169, 2023.
@inproceedings{10272824,
title = {Multi-Objective Fine-Grained Neural Architecture Search},
author = {Andrey Arhipov and Ivan Fomin},
url = {https://ieeexplore.ieee.org/abstract/document/10272824},
doi = {10.1109/RusAutoCon58002.2023.10272824},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {2023 International Russian Automation Conference (RusAutoCon)},
pages = {164-169},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
2462.
Lima, Rafael G. De; Freitas, Pedro Garcia; Lucafo, Giovani D.; Fioravanti, Vanessa; Seidel, Ismael; Penatti, Otávio A. B.
Neural Architecture Search for Tiny Detectors of Inter-beat Intervals Proceedings Article
In: 31st European Signal Processing Conference, EUSIPCO 2023, Helsinki, Finland, September 4-8, 2023, pp. 1085–1089, IEEE, 2023.
@inproceedings{DBLP:conf/eusipco/LimaFLFSP23,
title = {Neural Architecture Search for Tiny Detectors of Inter-beat Intervals},
author = {Rafael G. De Lima and Pedro Garcia Freitas and Giovani D. Lucafo and Vanessa Fioravanti and Ismael Seidel and Otávio A. B. Penatti},
url = {https://doi.org/10.23919/EUSIPCO58844.2023.10289754},
doi = {10.23919/EUSIPCO58844.2023.10289754},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {31st European Signal Processing Conference, EUSIPCO 2023, Helsinki,
Finland, September 4-8, 2023},
pages = {1085–1089},
publisher = {IEEE},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
2461.
Ghosh, Arjun; Jana, Nanda Dulal; Das, Swagatam; Mallipeddi, Rammohan
Two-Phase Evolutionary Convolutional Neural Network Architecture Search for Medical Image Classification Journal Article
In: IEEE Access, vol. 11, pp. 115280–115305, 2023.
@article{DBLP:journals/access/GhoshJDM23,
title = {Two-Phase Evolutionary Convolutional Neural Network Architecture Search for Medical Image Classification},
author = {Arjun Ghosh and Nanda Dulal Jana and Swagatam Das and Rammohan Mallipeddi},
url = {https://doi.org/10.1109/ACCESS.2023.3323705},
doi = {10.1109/ACCESS.2023.3323705},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {IEEE Access},
volume = {11},
pages = {115280–115305},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
2460.
Kulshrestha, Ankit; Lykov, Danylo; Safro, Ilya; Alexeev, Yuri
QArchSearch: A Scalable Quantum Architecture Search Package Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2310-07858,
title = {QArchSearch: A Scalable Quantum Architecture Search Package},
author = {Ankit Kulshrestha and Danylo Lykov and Ilya Safro and Yuri Alexeev},
url = {https://doi.org/10.48550/arXiv.2310.07858},
doi = {10.48550/ARXIV.2310.07858},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2310.07858},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2459.
Sun, Guocheng; Liu, Shitou; Shi, Chaojing; Liu, Xi; Guo, Qianjin
3DCNAS: A universal method for predicting the location of fluorescent organelles in living cells in three-dimensional space Journal Article
In: Experimental Cell Research, vol. 433, no. 2, pp. 113807, 2023, ISSN: 0014-4827.
@article{SUN2023113807,
title = {3DCNAS: A universal method for predicting the location of fluorescent organelles in living cells in three-dimensional space},
author = {Guocheng Sun and Shitou Liu and Chaojing Shi and Xi Liu and Qianjin Guo},
url = {https://www.sciencedirect.com/science/article/pii/S0014482723003555},
doi = {https://doi.org/10.1016/j.yexcr.2023.113807},
issn = {0014-4827},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {Experimental Cell Research},
volume = {433},
number = {2},
pages = {113807},
abstract = {Cellular biology research relies on microscopic imaging techniques for studying the complex structures and dynamic processes within cells. Fluorescence microscopy provides high sensitivity and subcellular resolution but has limitations such as photobleaching and sample preparation challenges. Transmission light microscopy offers a label-free alternative but lacks contrast for detailed interpretation. Deep learning methods have shown promise in analyzing cell images and extracting meaningful information. However, accurately learning and simulating diverse subcellular structures remain challenging. In this study, we propose a method named three-dimensional cell neural architecture search (3DCNAS) to predict subcellular structures of fluorescence using unlabeled transmitted light microscope images. By leveraging the automated search capability of differentiable neural architecture search (NAS), our method partially mitigates the issues of overfitting and underfitting caused by the distinct details of various subcellular structures. Furthermore, we apply our method to analyze cell dynamics in genome-edited human induced pluripotent stem cells during mitotic events. This allows us to study the functional roles of organelles and their involvement in cellular processes, contributing to a comprehensive understanding of cell biology and offering insights into disease pathogenesis.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Cellular biology research relies on microscopic imaging techniques for studying the complex structures and dynamic processes within cells. Fluorescence microscopy provides high sensitivity and subcellular resolution but has limitations such as photobleaching and sample preparation challenges. Transmission light microscopy offers a label-free alternative but lacks contrast for detailed interpretation. Deep learning methods have shown promise in analyzing cell images and extracting meaningful information. However, accurately learning and simulating diverse subcellular structures remain challenging. In this study, we propose a method named three-dimensional cell neural architecture search (3DCNAS) to predict subcellular structures of fluorescence using unlabeled transmitted light microscope images. By leveraging the automated search capability of differentiable neural architecture search (NAS), our method partially mitigates the issues of overfitting and underfitting caused by the distinct details of various subcellular structures. Furthermore, we apply our method to analyze cell dynamics in genome-edited human induced pluripotent stem cells during mitotic events. This allows us to study the functional roles of organelles and their involvement in cellular processes, contributing to a comprehensive understanding of cell biology and offering insights into disease pathogenesis.
2458.
Lv, Zeqiong; Qian, Chao; Sun, Yanan
Benchmarking Analysis of Evolutionary Neural Architecture Search Journal Article
In: IEEE Transactions on Evolutionary Computation, pp. 1-1, 2023.
@article{10286149,
title = {Benchmarking Analysis of Evolutionary Neural Architecture Search},
author = {Zeqiong Lv and Chao Qian and Yanan Sun},
url = {https://ieeexplore.ieee.org/abstract/document/10286149},
doi = {10.1109/TEVC.2023.3324852},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {IEEE Transactions on Evolutionary Computation},
pages = {1-1},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
2457.
Salehin, Imrus; Islam, Md. Shamiul; Saha, Pritom; Noman, S. M.; Tuni, Azra; Hasan, Md. Mehedi; Baten, Md. Abu
AutoML: A Systematic Review on Automated Machine Learning with Neural Architecture Search Journal Article
In: Journal of Information and Intelligence, 2023, ISSN: 2949-7159.
@article{SALEHIN2023,
title = {AutoML: A Systematic Review on Automated Machine Learning with Neural Architecture Search},
author = {Imrus Salehin and Md. Shamiul Islam and Pritom Saha and S. M. Noman and Azra Tuni and Md. Mehedi Hasan and Md. Abu Baten},
url = {https://www.sciencedirect.com/science/article/pii/S2949715923000604},
doi = {https://doi.org/10.1016/j.jiixd.2023.10.002},
issn = {2949-7159},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {Journal of Information and Intelligence},
abstract = {AutoML (Automated Machine Learning) is an emerging field that aims to automate the process of building machine learning models. AutoML emerged to increase productivity and efficiency by automating as much as possible the inefficient work that occurs while repeating this process whenever machine learning is applied. In particular, research has been conducted for a long time on technologies that can effectively develop high-quality models by minimizing the intervention of model developers in the process from data preprocessing to algorithm selection and tuning. In this semantic review research, we summarize the data processing requirements for AutoML approaches and provide a detailed explanation. We place greater emphasis on neural architecture search (NAS) as it currently represents a highly popular sub-topic within the field of AutoML. NAS methods use machine learning algorithms to search through a large space of possible architectures and find the one that performs best on a given task. We provide a summary of the performance achieved by representative NAS algorithms on the CIFAR-10, CIFAR-100, ImageNet and well-known benchmark datasets. Additionally, we delve into several noteworthy research directions in NAS methods including one/two-stage NAS, one-shot NAS and joint hyperparameter with architecture optimization. We discussed how the search space size and complexity in NAS can vary depending on the specific problem being addressed. To conclude, we examine several open problems (SOTA problems) within current AutoML methods that assure further investigation in future research.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
AutoML (Automated Machine Learning) is an emerging field that aims to automate the process of building machine learning models. AutoML emerged to increase productivity and efficiency by automating as much as possible the inefficient work that occurs while repeating this process whenever machine learning is applied. In particular, research has been conducted for a long time on technologies that can effectively develop high-quality models by minimizing the intervention of model developers in the process from data preprocessing to algorithm selection and tuning. In this semantic review research, we summarize the data processing requirements for AutoML approaches and provide a detailed explanation. We place greater emphasis on neural architecture search (NAS) as it currently represents a highly popular sub-topic within the field of AutoML. NAS methods use machine learning algorithms to search through a large space of possible architectures and find the one that performs best on a given task. We provide a summary of the performance achieved by representative NAS algorithms on the CIFAR-10, CIFAR-100, ImageNet and well-known benchmark datasets. Additionally, we delve into several noteworthy research directions in NAS methods including one/two-stage NAS, one-shot NAS and joint hyperparameter with architecture optimization. We discussed how the search space size and complexity in NAS can vary depending on the specific problem being addressed. To conclude, we examine several open problems (SOTA problems) within current AutoML methods that assure further investigation in future research.
2456.
Cao, Bin; Zhou, Zirun; Liu, Xin; Hossain, M. Shamim; Lv, Zhihan
Adaptive Multiobjective Evolutionary Neural Architecture Search for GANs Based on Two-Factor Cooperative Mutation Mechanism Proceedings Article
In: Proceedings of the 2023 Workshop on Advanced Multimedia Computing for Smart Manufacturing and Engineering, pp. 71–76, Association for Computing Machinery, Ottawa ON, Canada, 2023, ISBN: 9798400702730.
@inproceedings{10.1145/3606042.3616463,
title = {Adaptive Multiobjective Evolutionary Neural Architecture Search for GANs Based on Two-Factor Cooperative Mutation Mechanism},
author = {Bin Cao and Zirun Zhou and Xin Liu and M. Shamim Hossain and Zhihan Lv},
url = {https://doi.org/10.1145/3606042.3616463},
doi = {10.1145/3606042.3616463},
isbn = {9798400702730},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {Proceedings of the 2023 Workshop on Advanced Multimedia Computing for Smart Manufacturing and Engineering},
pages = {71–76},
publisher = {Association for Computing Machinery},
address = {Ottawa ON, Canada},
series = {AMC-SME '23},
abstract = {The automated design of generative adversarial networks (GAN) is currently being solved well by neural architecture search (NAS), although there are still some issues. One problem is the vast majority of NAS for GANs methods are only based on a single evaluation metric or a linear superposition of multiple evaluation metrics. Another problem is that the conventional evolutionary neural architecture search (ENAS) is unable to adjust its mutation probabilities in accordance with the NAS process, making it simple to settle into a local optimum. To address these issues, we firstly design a two-factor cooperative mutation mechanism that can control the mutation probability based on the current iteration rounds of the population, population fitness and other information. Secondly, we divide the evolutionary process into three stages based on the properties of NAS, so that the different stages can adaptively adjust the mutation probability according to the population state and the expected development goals. Finally, we incorporate multiple optimization objectives from GANs based on image generation tasks into ENAS. And we construct an adaptive multiobjective ENAS based on a two-factor cooperative mutation mechanism. We test and ablate our algorithm on the STL-10 and CIFAR-10 datasets, and the experimental results show that our method outperforms the majority of traditional NAS-GANs.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
The automated design of generative adversarial networks (GAN) is currently being solved well by neural architecture search (NAS), although there are still some issues. One problem is the vast majority of NAS for GANs methods are only based on a single evaluation metric or a linear superposition of multiple evaluation metrics. Another problem is that the conventional evolutionary neural architecture search (ENAS) is unable to adjust its mutation probabilities in accordance with the NAS process, making it simple to settle into a local optimum. To address these issues, we firstly design a two-factor cooperative mutation mechanism that can control the mutation probability based on the current iteration rounds of the population, population fitness and other information. Secondly, we divide the evolutionary process into three stages based on the properties of NAS, so that the different stages can adaptively adjust the mutation probability according to the population state and the expected development goals. Finally, we incorporate multiple optimization objectives from GANs based on image generation tasks into ENAS. And we construct an adaptive multiobjective ENAS based on a two-factor cooperative mutation mechanism. We test and ablate our algorithm on the STL-10 and CIFAR-10 datasets, and the experimental results show that our method outperforms the majority of traditional NAS-GANs.
2455.
Das, Richeek; Dooley, Samuel
Fairer and More Accurate Tabular Models Through NAS Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2310-12145,
title = {Fairer and More Accurate Tabular Models Through NAS},
author = {Richeek Das and Samuel Dooley},
url = {https://doi.org/10.48550/arXiv.2310.12145},
doi = {10.48550/ARXIV.2310.12145},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2310.12145},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2454.
Yao, Peng; Liao, Chao; Jia, Jiyuan; Tan, Jianchao; Chen, Bin; Song, Chengru; Zhang, Di
ASP: Automatic Selection of Proxy dataset for efficient AutoML Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2310-11478,
title = {ASP: Automatic Selection of Proxy dataset for efficient AutoML},
author = {Peng Yao and Chao Liao and Jiyuan Jia and Jianchao Tan and Bin Chen and Chengru Song and Di Zhang},
url = {https://doi.org/10.48550/arXiv.2310.11478},
doi = {10.48550/ARXIV.2310.11478},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2310.11478},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2453.
Yu, Caiyang; Wang, Yixi; Tang, Chenwei; Feng, Wentao; Lv, Jiancheng
EU-Net: Automatic U-Net neural architecture search with differential evolutionary algorithm for medical image segmentation Journal Article
In: Computers in Biology and Medicine, vol. 167, pp. 107579, 2023, ISSN: 0010-4825.
@article{YU2023107579,
title = {EU-Net: Automatic U-Net neural architecture search with differential evolutionary algorithm for medical image segmentation},
author = {Caiyang Yu and Yixi Wang and Chenwei Tang and Wentao Feng and Jiancheng Lv},
url = {https://www.sciencedirect.com/science/article/pii/S0010482523010442},
doi = {https://doi.org/10.1016/j.compbiomed.2023.107579},
issn = {0010-4825},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {Computers in Biology and Medicine},
volume = {167},
pages = {107579},
abstract = {Medical images are crucial in clinical practice, providing essential information for patient assessment and treatment planning. However, manual extraction of information from images is both time-consuming and prone to errors. The emergence of U-Net addresses this challenge by automating the segmentation of anatomical structures and pathological lesions in medical images, thereby significantly enhancing the accuracy of image interpretation and diagnosis. However, the performance of U-Net largely depends on its encoder–decoder structure, which requires researchers with knowledge of neural network architecture design and an in-depth understanding of medical images. In this paper, we propose an automatic U-Net Neural Architecture Search (NAS) algorithm using the differential evolutionary (DE) algorithm, named EU-Net, to segment critical information in medical images to assist physicians in diagnosis. Specifically, by presenting the variable-length strategy, the proposed EU-Net algorithm can sufficiently and automatically search for the neural network architecture without expertise. Moreover, the utilization of crossover, mutation, and selection strategies of DE takes account of the trade-off between exploration and exploitation in the search space. Finally, in the encoding and decoding phases of the proposed algorithm, different block-based and layer-based structures are introduced for architectural optimization. The proposed EU-Net algorithm is validated on two widely used medical datasets, i.e., CHAOS and BUSI, for image segmentation tasks. Extensive experimental results show that the proposed EU-Net algorithm outperforms the chosen peer competitors in both two datasets. In particular, compared to the original U-Net, our proposed method improves the metric mIou by at least 6%.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Medical images are crucial in clinical practice, providing essential information for patient assessment and treatment planning. However, manual extraction of information from images is both time-consuming and prone to errors. The emergence of U-Net addresses this challenge by automating the segmentation of anatomical structures and pathological lesions in medical images, thereby significantly enhancing the accuracy of image interpretation and diagnosis. However, the performance of U-Net largely depends on its encoder–decoder structure, which requires researchers with knowledge of neural network architecture design and an in-depth understanding of medical images. In this paper, we propose an automatic U-Net Neural Architecture Search (NAS) algorithm using the differential evolutionary (DE) algorithm, named EU-Net, to segment critical information in medical images to assist physicians in diagnosis. Specifically, by presenting the variable-length strategy, the proposed EU-Net algorithm can sufficiently and automatically search for the neural network architecture without expertise. Moreover, the utilization of crossover, mutation, and selection strategies of DE takes account of the trade-off between exploration and exploitation in the search space. Finally, in the encoding and decoding phases of the proposed algorithm, different block-based and layer-based structures are introduced for architectural optimization. The proposed EU-Net algorithm is validated on two widely used medical datasets, i.e., CHAOS and BUSI, for image segmentation tasks. Extensive experimental results show that the proposed EU-Net algorithm outperforms the chosen peer competitors in both two datasets. In particular, compared to the original U-Net, our proposed method improves the metric mIou by at least 6%.
2452.
Zou, Juan; Wu, Shenghong; Xia, Yizhang; Jiang, Weiwei; Wu, Zeping; Zheng, Jinhua
TS-ENAS: Two-Stage Evolution for Cell-based Network Architecture Search Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2310-09525,
title = {TS-ENAS: Two-Stage Evolution for Cell-based Network Architecture Search},
author = {Juan Zou and Shenghong Wu and Yizhang Xia and Weiwei Jiang and Zeping Wu and Jinhua Zheng},
url = {https://doi.org/10.48550/arXiv.2310.09525},
doi = {10.48550/ARXIV.2310.09525},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2310.09525},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2451.
Liu, Xiaoyun; Saxena, Divya; Cao, Jiannong; Zhao, Yuqing; Ruan, Penghui
MGAS: Multi-Granularity Architecture Search for Effective and Efficient Neural Networks Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2310-15074,
title = {MGAS: Multi-Granularity Architecture Search for Effective and Efficient Neural Networks},
author = {Xiaoyun Liu and Divya Saxena and Jiannong Cao and Yuqing Zhao and Penghui Ruan},
url = {https://doi.org/10.48550/arXiv.2310.15074},
doi = {10.48550/ARXIV.2310.15074},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2310.15074},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2450.
Jawahar, Ganesh; Abdul-Mageed, Muhammad; Lakshmanan, Laks V. S.; Ding, Dujian
LLM Performance Predictors are good initializers for Architecture Search Technical Report
2023.
@techreport{jawahar2023llm,
title = {LLM Performance Predictors are good initializers for Architecture Search},
author = {Ganesh Jawahar and Muhammad Abdul-Mageed and Laks V. S. Lakshmanan and Dujian Ding},
url = {https://arxiv.org/abs/2310.16712},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2449.
Chikwendu, Ijeoma Amuche; Zhang, Xiaoling; Agyemang, Isaac Osei; Mensah, Isaac Adjei; Ukwuoma, Chiagoziem Chima; Ejiyi, Chukwuebuka Joseph
A Comprehensive Survey on Deep Graph Representation Learning Methods Journal Article
In: J. Artif. Intell. Res., vol. 78, pp. 287–356, 2023.
@article{DBLP:journals/jair/ChikwenduZAMUE23,
title = {A Comprehensive Survey on Deep Graph Representation Learning Methods},
author = {Ijeoma Amuche Chikwendu and Xiaoling Zhang and Isaac Osei Agyemang and Isaac Adjei Mensah and Chiagoziem Chima Ukwuoma and Chukwuebuka Joseph Ejiyi},
url = {https://doi.org/10.1613/jair.1.14768},
doi = {10.1613/JAIR.1.14768},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {J. Artif. Intell. Res.},
volume = {78},
pages = {287–356},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
2448.
Liang, Yinan; Wang, Ziwei; Xu, Xiuwei; Tang, Yansong; Zhou, Jie; Lu, Jiwen
MCUFormer: Deploying Vision Tranformers on Microcontrollers with Limited Memory Technical Report
2023.
@techreport{liang2023mcuformer,
title = {MCUFormer: Deploying Vision Tranformers on Microcontrollers with Limited Memory},
author = {Yinan Liang and Ziwei Wang and Xiuwei Xu and Yansong Tang and Jie Zhou and Jiwen Lu},
url = {https://arxiv.org/abs/2310.16898},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2447.
Liu, Xiao; Yao, Chong; Chen, Hongyi; Xiang, Rui; Wu, Hao; Du, Peng; Yu, Zekuan; Liu, Weifan; Liu, Jie; Geng, Daoying
BTSC-TNAS: A neural architecture search-based transformer for brain tumor segmentation and classification Journal Article
In: Computerized Medical Imaging and Graphics, vol. 110, pp. 102307, 2023, ISSN: 0895-6111.
@article{LIU2023102307,
title = {BTSC-TNAS: A neural architecture search-based transformer for brain tumor segmentation and classification},
author = {Xiao Liu and Chong Yao and Hongyi Chen and Rui Xiang and Hao Wu and Peng Du and Zekuan Yu and Weifan Liu and Jie Liu and Daoying Geng},
url = {https://www.sciencedirect.com/science/article/pii/S0895611123001258},
doi = {https://doi.org/10.1016/j.compmedimag.2023.102307},
issn = {0895-6111},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {Computerized Medical Imaging and Graphics},
volume = {110},
pages = {102307},
abstract = {Glioblastoma (GBM), isolated brain metastasis (SBM), and primary central nervous system lymphoma (PCNSL) possess a high level of similarity in histomorphology and clinical manifestations on multimodal MRI. Such similarities have led to challenges in the clinical diagnosis of these three malignant tumors. However, many existing models solely focus on either the task of segmentation or classification, which limits the application of computer-aided diagnosis in clinical diagnosis and treatment. To solve this problem, we propose a multi-task learning transformer with neural architecture search (NAS) for brain tumor segmentation and classification (BTSC-TNAS). In the segmentation stage, we use a nested transformer U-shape network (NTU-NAS) with NAS to directly predict brain tumor masks from multi-modal MRI images. In the tumor classification stage, we use the multiscale features obtained from the encoder of NTU-NAS as the input features of the classification network (MSC-NET), which are integrated and corrected by the classification feature correction enhancement (CFCE) block to improve the accuracy of classification. The proposed BTSC-TNAS achieves an average Dice coefficient of 80.86% and 87.12% for the segmentation of tumor region and the maximum abnormal region in clinical data respectively. The model achieves a classification accuracy of 0.941. The experiments performed on the BraTS 2019 dataset show that the proposed BTSC-TNAS has excellent generalizability and can provide support for some challenging tasks in the diagnosis and treatment of brain tumors.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Glioblastoma (GBM), isolated brain metastasis (SBM), and primary central nervous system lymphoma (PCNSL) possess a high level of similarity in histomorphology and clinical manifestations on multimodal MRI. Such similarities have led to challenges in the clinical diagnosis of these three malignant tumors. However, many existing models solely focus on either the task of segmentation or classification, which limits the application of computer-aided diagnosis in clinical diagnosis and treatment. To solve this problem, we propose a multi-task learning transformer with neural architecture search (NAS) for brain tumor segmentation and classification (BTSC-TNAS). In the segmentation stage, we use a nested transformer U-shape network (NTU-NAS) with NAS to directly predict brain tumor masks from multi-modal MRI images. In the tumor classification stage, we use the multiscale features obtained from the encoder of NTU-NAS as the input features of the classification network (MSC-NET), which are integrated and corrected by the classification feature correction enhancement (CFCE) block to improve the accuracy of classification. The proposed BTSC-TNAS achieves an average Dice coefficient of 80.86% and 87.12% for the segmentation of tumor region and the maximum abnormal region in clinical data respectively. The model achieves a classification accuracy of 0.941. The experiments performed on the BraTS 2019 dataset show that the proposed BTSC-TNAS has excellent generalizability and can provide support for some challenging tasks in the diagnosis and treatment of brain tumors.
2446.
Zhou, Yuanding; Li, Xinran; Fang, Yaodong; Qin, Chuan
When Perceptual Authentication Hashing Meets Neural Architecture Search Proceedings Article
In: Proceedings of the 31st ACM International Conference on Multimedia, pp. 8975–8983, Association for Computing Machinery, Ottawa ON, Canada, 2023, ISBN: 9798400701085.
@inproceedings{10.1145/3581783.3612457,
title = {When Perceptual Authentication Hashing Meets Neural Architecture Search},
author = {Yuanding Zhou and Xinran Li and Yaodong Fang and Chuan Qin},
url = {https://doi.org/10.1145/3581783.3612457},
doi = {10.1145/3581783.3612457},
isbn = {9798400701085},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {Proceedings of the 31st ACM International Conference on Multimedia},
pages = {8975–8983},
publisher = {Association for Computing Machinery},
address = {Ottawa ON, Canada},
series = {MM '23},
abstract = {In recent years, many perceptual authentication hashing schemes have been proposed, especially for image content authentication. However, most of the schemes directly use the dataset of image processing during model training and evaluation, which is actually unreasonable due to the task difference. In this paper, we first propose a specialized dataset for perceptual authentication hashing of images (PAHI), and the image content-preserving manipulations used in this dataset are richer and more in line with realistic scenarios. Then, in order to achieve satisfactory perceptual robustness and discrimination capability of PAHI, we exploit the continuous neural architecture search (NAS) on the channel number and stack depth of the ConvNeXt architecture, and obtain two PAHI architectures i.e., NASRes and NASCoNt. The former has better overall performance, while the latter is better for some special manipulations such as image cropping and background overlap. Experimental results demonstrate that our architectures both can achieve competitive results compared with SOTA schemes, and the AUC areas are increased by 1.6 (NASCoNt) and 1.7 (NASRes), respectively.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
In recent years, many perceptual authentication hashing schemes have been proposed, especially for image content authentication. However, most of the schemes directly use the dataset of image processing during model training and evaluation, which is actually unreasonable due to the task difference. In this paper, we first propose a specialized dataset for perceptual authentication hashing of images (PAHI), and the image content-preserving manipulations used in this dataset are richer and more in line with realistic scenarios. Then, in order to achieve satisfactory perceptual robustness and discrimination capability of PAHI, we exploit the continuous neural architecture search (NAS) on the channel number and stack depth of the ConvNeXt architecture, and obtain two PAHI architectures i.e., NASRes and NASCoNt. The former has better overall performance, while the latter is better for some special manipulations such as image cropping and background overlap. Experimental results demonstrate that our architectures both can achieve competitive results compared with SOTA schemes, and the AUC areas are increased by 1.6 (NASCoNt) and 1.7 (NASRes), respectively.
2445.
Wang, Wenna; Zhuo, Tao; Zhang, Xiuwei; Sun, Mingjun; Yin, Hanlin; Xing, Yinghui; Zhang, Yanning
Automatic Network Architecture Search for RGB-D Semantic Segmentation Proceedings Article
In: Proceedings of the 31st ACM International Conference on Multimedia, pp. 3777–3786, Association for Computing Machinery, Ottawa ON, Canada, 2023, ISBN: 9798400701085.
@inproceedings{10.1145/3581783.3612288,
title = {Automatic Network Architecture Search for RGB-D Semantic Segmentation},
author = {Wenna Wang and Tao Zhuo and Xiuwei Zhang and Mingjun Sun and Hanlin Yin and Yinghui Xing and Yanning Zhang},
url = {https://doi.org/10.1145/3581783.3612288},
doi = {10.1145/3581783.3612288},
isbn = {9798400701085},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {Proceedings of the 31st ACM International Conference on Multimedia},
pages = {3777–3786},
publisher = {Association for Computing Machinery},
address = {Ottawa ON, Canada},
series = {MM '23},
abstract = {Recent RGB-D semantic segmentation networks are usually manually designed. However, due to limited human efforts and time costs, their performance might be inferior for complex scenarios. To address this issue, we propose the first Neural Architecture Search (NAS) method that designs the network automatically. Specifically, the target network consists of an encoder and a decoder. The encoder is designed with two independent branches, where each branch specializes in extracting features from RGB and depth images, respectively. The decoder fuses the features and generates the final segmentation result. Besides, for automatic network design, we design a grid-like network-level search space combined with a hierarchical cell-level search space. By further developing an effective gradient-based search strategy, the network structure with hierarchical cell architectures is discovered. Extensive results on two datasets show that the proposed method outperforms the state-of-the-art approaches, which achieves a mIoU score of 55.1% on the NYU-Depth v2 dataset and 50.3% on the SUN-RGBD dataset.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Recent RGB-D semantic segmentation networks are usually manually designed. However, due to limited human efforts and time costs, their performance might be inferior for complex scenarios. To address this issue, we propose the first Neural Architecture Search (NAS) method that designs the network automatically. Specifically, the target network consists of an encoder and a decoder. The encoder is designed with two independent branches, where each branch specializes in extracting features from RGB and depth images, respectively. The decoder fuses the features and generates the final segmentation result. Besides, for automatic network design, we design a grid-like network-level search space combined with a hierarchical cell-level search space. By further developing an effective gradient-based search strategy, the network structure with hierarchical cell architectures is discovered. Extensive results on two datasets show that the proposed method outperforms the state-of-the-art approaches, which achieves a mIoU score of 55.1% on the NYU-Depth v2 dataset and 50.3% on the SUN-RGBD dataset.
2444.
Zhang, Jiayi; Li, Weixin
Multi-Modal and Multi-Scale Temporal Fusion Architecture Search for Audio-Visual Video Parsing Proceedings Article
In: Proceedings of the 31st ACM International Conference on Multimedia, pp. 3328–3336, Association for Computing Machinery, Ottawa ON, Canada, 2023, ISBN: 9798400701085.
@inproceedings{10.1145/3581783.3611947,
title = {Multi-Modal and Multi-Scale Temporal Fusion Architecture Search for Audio-Visual Video Parsing},
author = {Jiayi Zhang and Weixin Li},
url = {https://doi.org/10.1145/3581783.3611947},
doi = {10.1145/3581783.3611947},
isbn = {9798400701085},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
booktitle = {Proceedings of the 31st ACM International Conference on Multimedia},
pages = {3328–3336},
publisher = {Association for Computing Machinery},
address = {Ottawa ON, Canada},
series = {MM '23},
abstract = {The weakly supervised audio-visual video parsing (AVVP) task aims to parse a video into a set of modality-wise events (i.e., audible, visible, or both), recognize categories of these events, and localize their temporal boundaries. Given the prevalence of audio-visual synchronous and asynchronous contents in multi-modal videos, it is crucial to capture and integrate the contextual events occurring at different moments and temporal scales. Although some researchers have made preliminary attempts at modeling event semantics with various temporal lengths, they mostly only perform a late fusion of multi-scale features across modalities. A comprehensive cross-modal and multi-scale temporal fusion strategy remains largely unexplored in the literature. To address this gap, we propose a novel framework named Audio-Visual Fusion Architecture Search (AVFAS) that can automatically find the optimal multi-scale temporal fusion strategy within and between modalities. Our framework generates a set of audio and visual features with distinct temporal scales and employs three modality-wise modules to search multi-scale feature selection and fusion strategies, jointly modeling modality-specific discriminative information. Furthermore, to enhance the alignment of audio-visual asynchrony, we introduce a Position- and Length-Adaptive Temporal Attention (PLATA) mechanism for cross-modal feature fusion. Extensive quantitative and qualitative experimental results demonstrate the effectiveness and efficiency of our framework.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
The weakly supervised audio-visual video parsing (AVVP) task aims to parse a video into a set of modality-wise events (i.e., audible, visible, or both), recognize categories of these events, and localize their temporal boundaries. Given the prevalence of audio-visual synchronous and asynchronous contents in multi-modal videos, it is crucial to capture and integrate the contextual events occurring at different moments and temporal scales. Although some researchers have made preliminary attempts at modeling event semantics with various temporal lengths, they mostly only perform a late fusion of multi-scale features across modalities. A comprehensive cross-modal and multi-scale temporal fusion strategy remains largely unexplored in the literature. To address this gap, we propose a novel framework named Audio-Visual Fusion Architecture Search (AVFAS) that can automatically find the optimal multi-scale temporal fusion strategy within and between modalities. Our framework generates a set of audio and visual features with distinct temporal scales and employs three modality-wise modules to search multi-scale feature selection and fusion strategies, jointly modeling modality-specific discriminative information. Furthermore, to enhance the alignment of audio-visual asynchrony, we introduce a Position- and Length-Adaptive Temporal Attention (PLATA) mechanism for cross-modal feature fusion. Extensive quantitative and qualitative experimental results demonstrate the effectiveness and efficiency of our framework.
2443.
Miranda, Thiago Z.; Sardinha, Diorge B.; Neri, Ferrante; Basgalupp, Márcio P.; Cerri, Ricardo
A Grammar-based multi-objective neuroevolutionary algorithm to generate fully convolutional networks with novel topologies Journal Article
In: Applied Soft Computing, vol. 149, pp. 110967, 2023, ISSN: 1568-4946.
@article{MIRANDA2023110967,
title = {A Grammar-based multi-objective neuroevolutionary algorithm to generate fully convolutional networks with novel topologies},
author = {Thiago Z. Miranda and Diorge B. Sardinha and Ferrante Neri and Márcio P. Basgalupp and Ricardo Cerri},
url = {https://www.sciencedirect.com/science/article/pii/S1568494623009857},
doi = {https://doi.org/10.1016/j.asoc.2023.110967},
issn = {1568-4946},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {Applied Soft Computing},
volume = {149},
pages = {110967},
abstract = {The design of complex and deep neural networks is often performed by identifying and combining building blocks and progressively selecting the most promising combination. Neuroevolution automates this process by employing evolutionary algorithms to guide the search. Within this field, grammar-based evolutionary algorithms have been demonstrated to be powerful tools to describe and thus encode complex neural architectures effectively. Following this trend, the present work proposes a novel grammar-based multi-objective neuroevolutionary for generating Fully Convolutional Networks. The proposed method, named Multi-Objective gRammatical Evolution for FUlly convolutional Networks (MOREFUN), includes a new efficient way to encode skip connections, facilitating the description of complex search spaces and the injection of domain knowledge in the search procedure, generation of fully convolutional networks, upsampling of lower-resolution inputs in multi-input layers, usage of multi-objective fitness, and inclusion of data augmentation and optimiser settings to the grammar. Our best networks outperformed previous grammar evolution algorithms, achieving 90.5% accuracy on CIFAR-10 without using transfer learning, ensembles, or test-time data augmentation. Our best models had 13.39±5.25 trainable parameters and the evolutionary process required 90 min per generation.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
The design of complex and deep neural networks is often performed by identifying and combining building blocks and progressively selecting the most promising combination. Neuroevolution automates this process by employing evolutionary algorithms to guide the search. Within this field, grammar-based evolutionary algorithms have been demonstrated to be powerful tools to describe and thus encode complex neural architectures effectively. Following this trend, the present work proposes a novel grammar-based multi-objective neuroevolutionary for generating Fully Convolutional Networks. The proposed method, named Multi-Objective gRammatical Evolution for FUlly convolutional Networks (MOREFUN), includes a new efficient way to encode skip connections, facilitating the description of complex search spaces and the injection of domain knowledge in the search procedure, generation of fully convolutional networks, upsampling of lower-resolution inputs in multi-input layers, usage of multi-objective fitness, and inclusion of data augmentation and optimiser settings to the grammar. Our best networks outperformed previous grammar evolution algorithms, achieving 90.5% accuracy on CIFAR-10 without using transfer learning, ensembles, or test-time data augmentation. Our best models had 13.39±5.25 trainable parameters and the evolutionary process required 90 min per generation.
2442.
King, Tobias; Zhou, Yexu; Röddiger, Tobias; Beigl, Michael
MicroNAS: Memory and Latency Constrained Hardware-Aware Neural Architecture Search for Time Series Classification on Microcontrollers Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2310-18384,
title = {MicroNAS: Memory and Latency Constrained Hardware-Aware Neural Architecture Search for Time Series Classification on Microcontrollers},
author = {Tobias King and Yexu Zhou and Tobias Röddiger and Michael Beigl},
url = {https://doi.org/10.48550/arXiv.2310.18384},
doi = {10.48550/ARXIV.2310.18384},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2310.18384},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2441.
Garcia-Garcia, Cosijopii; Morales-Reyes, Alicia; Escalante, Hugo Jair
Continuous Cartesian Genetic Programming based representation for Multi-Objective Neural Architecture Search Technical Report
2023.
@techreport{garciagarcia2023continuous,
title = {Continuous Cartesian Genetic Programming based representation for Multi-Objective Neural Architecture Search},
author = {Cosijopii Garcia-Garcia and Alicia Morales-Reyes and Hugo Jair Escalante},
url = {https://arxiv.org/abs/2306.02648},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2440.
Cao, Yufan; Zhang, Tunhou; Wen, Wei; Yan, Feng; Li, Hai; Chen, Yiran
Farthest Greedy Path Sampling for Two-shot Recommender Search Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2310-20705,
title = {Farthest Greedy Path Sampling for Two-shot Recommender Search},
author = {Yufan Cao and Tunhou Zhang and Wei Wen and Feng Yan and Hai Li and Yiran Chen},
url = {https://doi.org/10.48550/arXiv.2310.20705},
doi = {10.48550/ARXIV.2310.20705},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2310.20705},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2439.
Ding, Li; Zoghi, Masrour; Tennenholtz, Guy; Karimzadehgan, Maryam
Ever Evolving Evaluator (EV3): Towards Flexible and Reliable Meta-Optimization for Knowledge Distillation Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2310-18893,
title = {Ever Evolving Evaluator (EV3): Towards Flexible and Reliable Meta-Optimization for Knowledge Distillation},
author = {Li Ding and Masrour Zoghi and Guy Tennenholtz and Maryam Karimzadehgan},
url = {https://doi.org/10.48550/arXiv.2310.18893},
doi = {10.48550/ARXIV.2310.18893},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2310.18893},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2438.
Zhang, Tunhou; Wen, Wei; Fedorov, Igor; Liu, Xi; Zhang, Buyun; Han, Fangqiu; Chen, Wen-Yen; Han, Yiping; Yan, Feng; Li, Hai; Chen, Yiran
DistDNAS: Search Efficient Feature Interactions within 2 Hours Technical Report
2023.
@techreport{DBLP:journals/corr/abs-2311-00231,
title = {DistDNAS: Search Efficient Feature Interactions within 2 Hours},
author = {Tunhou Zhang and Wei Wen and Igor Fedorov and Xi Liu and Buyun Zhang and Fangqiu Han and Wen-Yen Chen and Yiping Han and Feng Yan and Hai Li and Yiran Chen},
url = {https://doi.org/10.48550/arXiv.2311.00231},
doi = {10.48550/ARXIV.2311.00231},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {CoRR},
volume = {abs/2311.00231},
keywords = {},
pubstate = {published},
tppubtype = {techreport}
}
2437.
Sun, Pingping; Fan, Shijie; Li, Shaochuan; Zhao, Yingwei; Lu, Chang; Wong, Ka-Chun; Li, Xiangtao
Automated exploitation of deep learning for cancer patient stratification across multiple types Journal Article
In: Bioinformatics, vol. 39, no. 11, pp. btad654, 2023, ISSN: 1367-4811.
@article{10.1093/bioinformatics/btad654,
title = {Automated exploitation of deep learning for cancer patient stratification across multiple types},
author = {Pingping Sun and Shijie Fan and Shaochuan Li and Yingwei Zhao and Chang Lu and Ka-Chun Wong and Xiangtao Li},
url = {https://doi.org/10.1093/bioinformatics/btad654},
doi = {10.1093/bioinformatics/btad654},
issn = {1367-4811},
year = {2023},
date = {2023-01-01},
urldate = {2023-01-01},
journal = {Bioinformatics},
volume = {39},
number = {11},
pages = {btad654},
abstract = {Recent frameworks based on deep learning have been developed to identify cancer subtypes from high-throughput gene expression profiles. Unfortunately, the performance of deep learning is highly dependent on its neural network architectures which are often hand-crafted with expertise in deep neural networks, meanwhile, the optimization and adjustment of the network are usually costly and time consuming.To address such limitations, we proposed a fully automated deep neural architecture search model for diagnosing consensus molecular subtypes from gene expression data (DNAS). The proposed model uses ant colony algorithm, one of the heuristic swarm intelligence algorithms, to search and optimize neural network architecture, and it can automatically find the optimal deep learning model architecture for cancer diagnosis in its search space. We validated DNAS on eight colorectal cancer datasets, achieving the average accuracy of 95.48%, the average specificity of 98.07%, and the average sensitivity of 96.24%, respectively. Without the loss of generality, we investigated the general applicability of DNAS further on other cancer types from different platforms including lung cancer and breast cancer, and DNAS achieved an area under the curve of 95% and 96%, respectively. In addition, we conducted gene ontology enrichment and pathological analysis to reveal interesting insights into cancer subtype identification and characterization across multiple cancer types.The source code and data can be downloaded from https://github.com/userd113/DNAS-main. And the web server of DNAS is publicly accessible at 119.45.145.120:5001.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Recent frameworks based on deep learning have been developed to identify cancer subtypes from high-throughput gene expression profiles. Unfortunately, the performance of deep learning is highly dependent on its neural network architectures which are often hand-crafted with expertise in deep neural networks, meanwhile, the optimization and adjustment of the network are usually costly and time consuming.To address such limitations, we proposed a fully automated deep neural architecture search model for diagnosing consensus molecular subtypes from gene expression data (DNAS). The proposed model uses ant colony algorithm, one of the heuristic swarm intelligence algorithms, to search and optimize neural network architecture, and it can automatically find the optimal deep learning model architecture for cancer diagnosis in its search space. We validated DNAS on eight colorectal cancer datasets, achieving the average accuracy of 95.48%, the average specificity of 98.07%, and the average sensitivity of 96.24%, respectively. Without the loss of generality, we investigated the general applicability of DNAS further on other cancer types from different platforms including lung cancer and breast cancer, and DNAS achieved an area under the curve of 95% and 96%, respectively. In addition, we conducted gene ontology enrichment and pathological analysis to reveal interesting insights into cancer subtype identification and characterization across multiple cancer types.The source code and data can be downloaded from https://github.com/userd113/DNAS-main. And the web server of DNAS is publicly accessible at 119.45.145.120:5001.