
@article{pak_agentic_2026,
	title = {Agentic additive manufacturing alloy evaluation},
	volume = {17},
	issn = {2772-3690},
	url = {https://www.sciencedirect.com/science/article/pii/S2772369026000022},
	doi = {10.1016/j.addlet.2026.100355},
	abstract = {Agentic systems enable the intelligent use of research tooling, augmenting a researcher’s ability to investigate and propose novel solutions to existing problems. Within Additive Manufacturing (AM), alloy selection and evaluation remains a complex challenge, often requiring expertise in the various domains of materials science, thermodynamic simulations, and experimental analysis. Large Language Model (LLM) enabled agents can facilitate this endeavor by utilizing their extensive knowledge base to dispatch tool calls via Model Context Protocol (MCP) to perform actions such as thermophysical property diagram calculations and lack of fusion process map generation. In addition, the multi-agent system can effectively reason through complex user prompts and provide analysis on the lack of fusion process window of common alloys such as SS316L and IN718 along with proposed composition variants of known alloys. These agents can dynamically adjust their task trajectory to the outcomes of tool call results, effectively enabling autonomous decision-making in practical environments. This work aims to showcase the benefits of adopting a LLM enabled multi-agent system to automate and accelerate the task of evaluating proposed additive manufacturing alloys, both novel and known.},
	urldate = {2026-03-23},
	journal = {Additive Manufacturing Letters},
	author = {Pak, Peter and Chandrasekhar, Achuth and Barati Farimani, Amir},
	month = apr,
	year = {2026},
	keywords = {Additive Manufacturing, Agents, Alloy Discovery, CALPHAD, Model Context Protocol, Thermo-Calc, Tool Calling, laser powder bed fusion},
	pages = {100355},
}

@misc{noauthor_algorithmicsuperintelligenceopenevolve_2026,
	title = {algorithmicsuperintelligence/openevolve},
	copyright = {Apache-2.0},
	url = {https://github.com/algorithmicsuperintelligence/openevolve},
	abstract = {Open-source implementation of AlphaEvolve},
	urldate = {2026-03-23},
	publisher = {Algorithmic SuperIntelligence Labs},
	month = mar,
	year = {2026},
	note = {original-date: 2025-05-15T11:46:52Z},
	keywords = {alpha-evolve, alphacode, alphaevolve, coding-agent, deepmind, deepmind-lab, discovery, distributed-evolutionary-algorithms, evolutionary-algorithms, evolutionary-computation, genetic-algorithm, genetic-algorithms, iterative-methods, iterative-refinement, llm-engineering, llm-ensemble, llm-inference, openevolve, optimize},
}

@misc{novikov_alphaevolve_2025,
	title = {{AlphaEvolve}: {A} coding agent for scientific and algorithmic discovery},
	shorttitle = {{AlphaEvolve}},
	url = {http://arxiv.org/abs/2506.13131},
	doi = {10.48550/arXiv.2506.13131},
	abstract = {In this white paper, we present AlphaEvolve, an evolutionary coding agent that substantially enhances capabilities of state-of-the-art LLMs on highly challenging tasks such as tackling open scientific problems or optimizing critical pieces of computational infrastructure. AlphaEvolve orchestrates an autonomous pipeline of LLMs, whose task is to improve an algorithm by making direct changes to the code. Using an evolutionary approach, continuously receiving feedback from one or more evaluators, AlphaEvolve iteratively improves the algorithm, potentially leading to new scientific and practical discoveries. We demonstrate the broad applicability of this approach by applying it to a number of important computational problems. When applied to optimizing critical components of large-scale computational stacks at Google, AlphaEvolve developed a more efficient scheduling algorithm for data centers, found a functionally equivalent simplification in the circuit design of hardware accelerators, and accelerated the training of the LLM underpinning AlphaEvolve itself. Furthermore, AlphaEvolve discovered novel, provably correct algorithms that surpass state-of-the-art solutions on a spectrum of problems in mathematics and computer science, significantly expanding the scope of prior automated discovery methods (Romera-Paredes et al., 2023). Notably, AlphaEvolve developed a search algorithm that found a procedure to multiply two \$4 {\textbackslash}times 4\$ complex-valued matrices using \$48\$ scalar multiplications; offering the first improvement, after 56 years, over Strassen's algorithm in this setting. We believe AlphaEvolve and coding agents like it can have a significant impact in improving solutions of problems across many areas of science and computation.},
	urldate = {2026-03-19},
	publisher = {arXiv},
	author = {Novikov, Alexander and Vũ, Ngân and Eisenberger, Marvin and Dupont, Emilien and Huang, Po-Sen and Wagner, Adam Zsolt and Shirobokov, Sergey and Kozlovskii, Borislav and Ruiz, Francisco J. R. and Mehrabian, Abbas and Kumar, M. Pawan and See, Abigail and Chaudhuri, Swarat and Holland, George and Davies, Alex and Nowozin, Sebastian and Kohli, Pushmeet and Balog, Matej},
	month = jun,
	year = {2025},
	note = {arXiv:2506.13131 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Machine Learning, Computer Science - Neural and Evolutionary Computing},
}

@misc{ellenberg_generative_2025,
	title = {Generative {Modeling} for {Mathematical} {Discovery}},
	url = {http://arxiv.org/abs/2503.11061},
	doi = {10.48550/arXiv.2503.11061},
	abstract = {We present a new implementation of the LLM-driven genetic algorithm \{{\textbackslash}it funsearch\}, whose aim is to generate examples of interest to mathematicians and which has already had some success in problems in extremal combinatorics. Our implementation is designed to be useful in practice for working mathematicians; it does not require expertise in machine learning or access to high-performance computing resources. Applying \{{\textbackslash}it funsearch\} to a new problem involves modifying a small segment of Python code and selecting a large language model (LLM) from one of many third-party providers. We benchmarked our implementation on three different problems, obtaining metrics that may inform applications of \{{\textbackslash}it funsearch\} to new problems. Our results demonstrate that \{{\textbackslash}it funsearch\} successfully learns in a variety of combinatorial and number-theoretic settings, and in some contexts learns principles that generalize beyond the problem originally trained on.},
	urldate = {2026-03-19},
	publisher = {arXiv},
	author = {Ellenberg, Jordan S. and Fraser-Taliente, Cristofero S. and Harvey, Thomas R. and Srivastava, Karan and Sutherland, Andrew V.},
	month = mar,
	year = {2025},
	note = {arXiv:2503.11061 [cs]},
	keywords = {Computer Science - Machine Learning, Mathematics - Combinatorics},
}

@misc{muennighoff_s1_2025,
	title = {s1: {Simple} test-time scaling},
	shorttitle = {s1},
	url = {http://arxiv.org/abs/2501.19393},
	doi = {10.48550/arXiv.2501.19393},
	abstract = {Test-time scaling is a promising new approach to language modeling that uses extra test-time compute to improve performance. Recently, OpenAI's o1 model showed this capability but did not publicly share its methodology, leading to many replication efforts. We seek the simplest approach to achieve test-time scaling and strong reasoning performance. First, we curate a small dataset s1K of 1,000 questions paired with reasoning traces relying on three criteria we validate through ablations: difficulty, diversity, and quality. Second, we develop budget forcing to control test-time compute by forcefully terminating the model's thinking process or lengthening it by appending "Wait" multiple times to the model's generation when it tries to end. This can lead the model to double-check its answer, often fixing incorrect reasoning steps. After supervised finetuning the Qwen2.5-32B-Instruct language model on s1K and equipping it with budget forcing, our model s1-32B exceeds o1-preview on competition math questions by up to 27\% (MATH and AIME24). Further, scaling s1-32B with budget forcing allows extrapolating beyond its performance without test-time intervention: from 50\% to 57\% on AIME24. Our model, data, and code are open-source at https://github.com/simplescaling/s1},
	urldate = {2026-03-19},
	publisher = {arXiv},
	author = {Muennighoff, Niklas and Yang, Zitong and Shi, Weijia and Li, Xiang Lisa and Fei-Fei, Li and Hajishirzi, Hannaneh and Zettlemoyer, Luke and Liang, Percy and Candès, Emmanuel and Hashimoto, Tatsunori},
	month = mar,
	year = {2025},
	note = {arXiv:2501.19393 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computation and Language, Computer Science - Machine Learning},
}

@article{jegou_product_2011,
	title = {Product {Quantization} for {Nearest} {Neighbor} {Search}},
	volume = {33},
	issn = {1939-3539},
	url = {https://ieeexplore.ieee.org/document/5432202},
	doi = {10.1109/TPAMI.2010.57},
	abstract = {This paper introduces a product quantization-based approach for approximate nearest neighbor search. The idea is to decompose the space into a Cartesian product of low-dimensional subspaces and to quantize each subspace separately. A vector is represented by a short code composed of its subspace quantization indices. The euclidean distance between two vectors can be efficiently estimated from their codes. An asymmetric version increases precision, as it computes the approximate distance between a vector and a code. Experimental results show that our approach searches for nearest neighbors efficiently, in particular in combination with an inverted file system. Results for SIFT and GIST image descriptors show excellent search accuracy, outperforming three state-of-the-art approaches. The scalability of our approach is validated on a data set of two billion vectors.},
	number = {1},
	urldate = {2026-02-18},
	journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
	author = {Jégou, Herve and Douze, Matthijs and Schmid, Cordelia},
	month = jan,
	year = {2011},
	keywords = {Electronic mail, Euclidean distance, File systems, High-dimensional indexing, Image databases, Indexing, Nearest neighbor searches, Neural networks, Permission, Quantization, Scalability, approximate search., image indexing, very large databases},
	pages = {117--128},
}

@inproceedings{weber_quantitative_1998,
	title = {A {Quantitative} {Analysis} and {Performance} {Study} for {Similarity}-{Search} {Methods} in {High}-{Dimensional} {Spaces}},
	url = {https://www.semanticscholar.org/paper/A-Quantitative-Analysis-and-Performance-Study-for-Weber-Schek/63eaeb0c48175065ffd096aad10aed712c6d7bbb},
	abstract = {For similarity search in high-dimensional vector spaces (or ‘HDVSs’), researchers have proposed a number of new methods (or adaptations of existing methods) based, in the main, on data-space partitioning. However, the performance of these methods generally degrades as dimensionality increases. Although this phenomenon-known as the ‘dimensional curse’-is well known, little or no quantitative a.nalysis of the phenomenon is available. In this paper, we provide a detailed analysis of partitioning and clustering techniques for similarity search in HDVSs. We show formally that these methods exhibit linear complexity at high dimensionality, and that existing methods are outperformed on average by a simple sequential scan if the number of dimensions exceeds around 10. Consequently, we come up with an alternative organization based on approximations to make the unavoidable sequential scan as fast as possible. We describe a simple vector approximation scheme, called VA-file, and report on an experimental evaluation of this and of two tree-based index methods (an R*-tree and an X-tree).},
	urldate = {2026-02-18},
	author = {Weber, R. and Schek, H. and Blott, Stephen},
	month = aug,
	year = {1998},
}

@misc{johnson_billion-scale_2017,
	title = {Billion-scale similarity search with {GPUs}},
	copyright = {arXiv.org perpetual, non-exclusive license},
	url = {https://arxiv.org/abs/1702.08734},
	doi = {10.48550/ARXIV.1702.08734},
	abstract = {Similarity search finds application in specialized database systems handling complex data such as images or videos, which are typically represented by high-dimensional features and require specific indexing structures. This paper tackles the problem of better utilizing GPUs for this task. While GPUs excel at data-parallel tasks, prior approaches are bottlenecked by algorithms that expose less parallelism, such as k-min selection, or make poor use of the memory hierarchy. We propose a design for k-selection that operates at up to 55\% of theoretical peak performance, enabling a nearest neighbor implementation that is 8.5x faster than prior GPU state of the art. We apply it in different similarity search scenarios, by proposing optimized design for brute-force, approximate and compressed-domain search based on product quantization. In all these setups, we outperform the state of the art by large margins. Our implementation enables the construction of a high accuracy k-NN graph on 95 million images from the Yfcc100M dataset in 35 minutes, and of a graph connecting 1 billion vectors in less than 12 hours on 4 Maxwell Titan X GPUs. We have open-sourced our approach for the sake of comparison and reproducibility.},
	urldate = {2026-02-18},
	publisher = {arXiv},
	author = {Johnson, Jeff and Douze, Matthijs and Jégou, Hervé},
	year = {2017},
	note = {Version Number: 1},
	keywords = {Computer Vision and Pattern Recognition (cs.CV), Data Structures and Algorithms (cs.DS), Databases (cs.DB), FOS: Computer and information sciences, Information Retrieval (cs.IR)},
}

@misc{douze_faiss_2024,
	title = {The {Faiss} library},
	copyright = {Creative Commons Attribution 4.0 International},
	url = {https://arxiv.org/abs/2401.08281},
	doi = {10.48550/ARXIV.2401.08281},
	abstract = {Vector databases typically manage large collections of embedding vectors. Currently, AI applications are growing rapidly, and so is the number of embeddings that need to be stored and indexed. The Faiss library is dedicated to vector similarity search, a core functionality of vector databases. Faiss is a toolkit of indexing methods and related primitives used to search, cluster, compress and transform vectors. This paper describes the trade-off space of vector search and the design principles of Faiss in terms of structure, approach to optimization and interfacing. We benchmark key features of the library and discuss a few selected applications to highlight its broad applicability.},
	urldate = {2026-02-18},
	publisher = {arXiv},
	author = {Douze, Matthijs and Guzhva, Alexandr and Deng, Chengqi and Johnson, Jeff and Szilvasy, Gergely and Mazaré, Pierre-Emmanuel and Lomeli, Maria and Hosseini, Lucas and Jégou, Hervé},
	year = {2024},
	note = {Version Number: 4},
	keywords = {Computer Vision and Pattern Recognition (cs.CV), FOS: Computer and information sciences, Machine Learning (cs.LG), Software Engineering (cs.SE)},
}

@article{yang_swe-agent_2024,
	title = {{SWE}-agent: {Agent}-{Computer} {Interfaces} {Enable} {Automated} {Software} {Engineering}},
	volume = {37},
	shorttitle = {{SWE}-agent},
	url = {https://proceedings.neurips.cc/paper_files/paper/2024/hash/5a7c947568c1b1328ccc5230172e1e7c-Abstract-Conference.html},
	doi = {10.52202/079017-1601},
	language = {en},
	urldate = {2026-03-13},
	journal = {Advances in Neural Information Processing Systems},
	author = {Yang, John and Jimenez, Carlos E. and Wettig, Alexander and Lieret, Kilian and Yao, Shunyu and Narasimhan, Karthik and Press, Ofir},
	month = dec,
	year = {2024},
	pages = {50528--50652},
}

@misc{xia_agentless_2024,
	title = {Agentless: {Demystifying} {LLM}-based {Software} {Engineering} {Agents}},
	shorttitle = {Agentless},
	url = {http://arxiv.org/abs/2407.01489},
	doi = {10.48550/arXiv.2407.01489},
	abstract = {Recent advancements in large language models (LLMs) have significantly advanced the automation of software development tasks, including code synthesis, program repair, and test generation. More recently, researchers and industry practitioners have developed various autonomous LLM agents to perform end-to-end software development tasks. These agents are equipped with the ability to use tools, run commands, observe feedback from the environment, and plan for future actions. However, the complexity of these agent-based approaches, together with the limited abilities of current LLMs, raises the following question: Do we really have to employ complex autonomous software agents? To attempt to answer this question, we build Agentless -- an agentless approach to automatically solve software development problems. Compared to the verbose and complex setup of agent-based approaches, Agentless employs a simplistic three-phase process of localization, repair, and patch validation, without letting the LLM decide future actions or operate with complex tools. Our results on the popular SWE-bench Lite benchmark show that surprisingly the simplistic Agentless is able to achieve both the highest performance (32.00\%, 96 correct fixes) and low cost (\$0.70) compared with all existing open-source software agents! Furthermore, we manually classified the problems in SWE-bench Lite and found problems with exact ground truth patch or insufficient/misleading issue descriptions. As such, we construct SWE-bench Lite-S by excluding such problematic issues to perform more rigorous evaluation and comparison. Our work highlights the current overlooked potential of a simple, interpretable technique in autonomous software development. We hope Agentless will help reset the baseline, starting point, and horizon for autonomous software agents, and inspire future work along this crucial direction.},
	urldate = {2026-03-13},
	publisher = {arXiv},
	author = {Xia, Chunqiu Steven and Deng, Yinlin and Dunn, Soren and Zhang, Lingming},
	month = oct,
	year = {2024},
	note = {arXiv:2407.01489 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computation and Language, Computer Science - Machine Learning, Computer Science - Software Engineering},
}

@misc{mu_experepair_2025,
	title = {{EXPEREPAIR}: {Dual}-{Memory} {Enhanced} {LLM}-based {Repository}-{Level} {Program} {Repair}},
	shorttitle = {{EXPEREPAIR}},
	url = {http://arxiv.org/abs/2506.10484},
	doi = {10.48550/arXiv.2506.10484},
	abstract = {Automatically repairing software issues remains a fundamental challenge at the intersection of software engineering and AI. Although recent advancements in Large Language Models (LLMs) have demonstrated potential for repository-level repair tasks, current methodologies exhibit two notable limitations: (1) they often address issues in isolation, neglecting to incorporate insights from previously resolved issues, and (2) they rely on static and rigid prompting strategies, which constrain their ability to generalize across diverse and evolving issue scenarios. Inspired by the dual memory systems of human cognition, where episodic and semantic memories work synergistically to support human reasoning and decision-making, we propose ExpeRepair, a novel LLM-based approach that continuously learns from historical repair experiences through dual-channel knowledge accumulation. ExpeRepair organizes historical repair experiences into two complementary memories: an episodic memory that stores concrete repair demonstrations, and a semantic memory that encodes abstract reflective insights. At inference time, ExpeRepair activates both memory systems by retrieving relevant demonstrations from episodic memory and recalling high-level repair insights from semantic memory. It further enhances adaptability through dynamic prompt composition, synergistically integrating both memory types to replace static prompts with context-aware, experience-driven prompts. Experiments on the SWE-bench Lite benchmark demonstrate that ExpeRepair achieves a pass@1 score of 49.3\% with Claude 3.7 Sonnet, outperforming all state-of-the-art open-source methods.},
	urldate = {2026-03-13},
	publisher = {arXiv},
	author = {Mu, Fangwen and Wang, Junjie and Shi, Lin and Wang, Song and Li, Shoubin and Wang, Qing},
	month = jun,
	year = {2025},
	note = {arXiv:2506.10484 [cs]},
	keywords = {Computer Science - Software Engineering},
}

@misc{wang_openhands_2025,
	title = {{OpenHands}: {An} {Open} {Platform} for {AI} {Software} {Developers} as {Generalist} {Agents}},
	shorttitle = {{OpenHands}},
	url = {http://arxiv.org/abs/2407.16741},
	doi = {10.48550/arXiv.2407.16741},
	abstract = {Software is one of the most powerful tools that we humans have at our disposal; it allows a skilled programmer to interact with the world in complex and profound ways. At the same time, thanks to improvements in large language models (LLMs), there has also been a rapid development in AI agents that interact with and affect change in their surrounding environments. In this paper, we introduce OpenHands (f.k.a. OpenDevin), a platform for the development of powerful and flexible AI agents that interact with the world in similar ways to those of a human developer: by writing code, interacting with a command line, and browsing the web. We describe how the platform allows for the implementation of new agents, safe interaction with sandboxed environments for code execution, coordination between multiple agents, and incorporation of evaluation benchmarks. Based on our currently incorporated benchmarks, we perform an evaluation of agents over 15 challenging tasks, including software engineering (e.g., SWE-BENCH) and web browsing (e.g., WEBARENA), among others. Released under the permissive MIT license, OpenHands is a community project spanning academia and industry with more than 2.1K contributions from over 188 contributors.},
	urldate = {2026-03-13},
	publisher = {arXiv},
	author = {Wang, Xingyao and Li, Boxuan and Song, Yufan and Xu, Frank F. and Tang, Xiangru and Zhuge, Mingchen and Pan, Jiayi and Song, Yueqi and Li, Bowen and Singh, Jaskirat and Tran, Hoang H. and Li, Fuqiang and Ma, Ren and Zheng, Mingzhang and Qian, Bill and Shao, Yanjun and Muennighoff, Niklas and Zhang, Yizhe and Hui, Binyuan and Lin, Junyang and Brennan, Robert and Peng, Hao and Ji, Heng and Neubig, Graham},
	month = apr,
	year = {2025},
	note = {arXiv:2407.16741 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computation and Language, Computer Science - Software Engineering},
}

@misc{han_tdflow_2026,
	title = {{TDFlow}: {Agentic} {Workflows} for {Test} {Driven} {Development}},
	shorttitle = {{TDFlow}},
	url = {http://arxiv.org/abs/2510.23761},
	doi = {10.48550/arXiv.2510.23761},
	abstract = {We introduce TDFlow, a novel test-driven agentic workflow that frames repository-scale software engineering as a test-resolution task, specifically designed to solve human-written tests. Given a set of tests, TDFlow repeatedly proposes, revises, and debugs repository-scale patches using precisely engineered sub-agents and tightly constrained tools. The workflow decomposes software engineering program repair into four components governed by respective sub-agents. This simple, forced decoupling of patch proposing, debugging, patch revision, and optional test generation (1) reduces long-context burden on any individual sub-agent, (2) focuses each sub-agent on specific, pre-defined sub-tasks, and (3) allows for specialized performance improvement on specific sub-tasks. When provided human-written tests, TDFlow attains 88.8\% pass rate on SWE-Bench Lite (an absolute improvement of 27.8\% over the next best system) and 94.3\% on SWE-Bench Verified. Manual inspection of the 800 TDFlow runs within SWE-Bench Lite and Verified uncover only 7 instances of test hacking, which were subsequently counted as failures. Furthermore, we show that the primary obstacle to human-level software engineering performance lies within writing successful reproduction tests. We envision a human-LLM interactive system powered by TDFlow where human developers write tests solved by LLM systems. Together, these results indicate that modern LLMs, when embedded in a narrowly engineered, test-driven workflow, already achieve human-level test resolution -- with the final frontier for fully autonomous repository repair being the accurate generation of valid reproduction tests.},
	urldate = {2026-03-13},
	publisher = {arXiv},
	author = {Han, Kevin and Maddikayala, Siddharth and Knappe, Tim and Patel, Om and Liao, Austen and Farimani, Amir Barati},
	month = jan,
	year = {2026},
	note = {arXiv:2510.23761 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Multiagent Systems, Computer Science - Software Engineering},
}

@article{george_structured_2004,
	series = {Special {Issue} on {Software} {Engineering},  {Applications}, {Practices} and {Tools} from the {ACM} {Symposium} on {Applied} {Computing} 2003},
	title = {A structured experiment of test-driven development},
	volume = {46},
	issn = {0950-5849},
	url = {https://www.sciencedirect.com/science/article/pii/S0950584903002040},
	doi = {10.1016/j.infsof.2003.09.011},
	abstract = {Test Driven Development (TDD) is a software development practice in which unit test cases are incrementally written prior to code implementation. We ran a set of structured experiments with 24 professional pair programmers. One group developed a small Java program using TDD while the other (control group), used a waterfall-like approach. Experimental results, subject to external validity concerns, tend to indicate that TDD programmers produce higher quality code because they passed 18\% more functional black-box test cases. However, the TDD programmers took 16\% more time. Statistical analysis of the results showed that a moderate statistical correlation existed between time spent and the resulting quality. Lastly, the programmers in the control group often did not write the required automated test cases after completing their code. Hence it could be perceived that waterfall-like approaches do not encourage adequate testing. This intuitive observation supports the perception that TDD has the potential for increasing the level of unit testing in the software industry.},
	number = {5},
	urldate = {2026-03-13},
	journal = {Information and Software Technology},
	author = {George, Boby and Williams, Laurie},
	month = apr,
	year = {2004},
	keywords = {Agile methodologies, Extreme programming, Software engineering, Test driven development},
	pages = {337--342},
}

@misc{noauthor_fdm_nodate,
	title = {{FDM} {3D} {Printing} {Defect} {Dataset}},
	url = {https://www.kaggle.com/datasets/wengmhu/fdm-3d-printing-defect-dataset},
	abstract = {Five types of defect image datasets for 3D printing},
	language = {en},
	urldate = {2026-03-08},
}

@article{hu_real-time_2024,
	title = {Real-time defect detection for {FFF} {3D} printing using lightweight model deployment},
	volume = {134},
	issn = {1433-3015},
	url = {https://doi.org/10.1007/s00170-024-14452-4},
	doi = {10.1007/s00170-024-14452-4},
	abstract = {FFF 3D printing is one of the most widely used additive manufacturing methods, bringing great convenience to production manufacturing. However, various printing defects may occur during the printing process due to human factors or printer-related issues. Timely detection of defects and halting printing becomes a scenario of significant practical importance. This paper first analyzes the causes of the five most common defects in FFF 3D printing, and a defect dataset is created by deliberately designing defects. Subsequently, a real-time defect detection system for FFF 3D printing, based on an improved YOLOv8 detection head, is developed. By employing an optimization method using Group Convolution to share parameters, the detection head is lightweight, resulting in better model performance. Experimental results demonstrate that the mAP50 of the improved YOLOv8 model reaches 97.5\%, with an 18.1\% increase in FPS and a 32.9\% reduction in GFLOPs. This enhancement maintains comparable detection accuracy to the original model while achieving faster detection speed and lower computational requirements. The improved model is integrated into the detection system as the detection model, and through testing, the real-time detection system promptly and accurately identifies and alerts any occurring defects. The practical significance of this system lies in its ability to enhance production efficiency, reduce resource wastage due to defective printing, and improve product quality and manufacturing safety, thereby providing strong support for the application of visual inspection technology in FFF 3D printing.},
	language = {en},
	number = {9},
	urldate = {2026-03-08},
	journal = {The International Journal of Advanced Manufacturing Technology},
	author = {Hu, WenJing and Chen, Chang and Su, Shaohui and Zhang, Jian and Zhu, An},
	month = oct,
	year = {2024},
	keywords = {FFF 3D printing, Group Convolution, Lightweight detection head, Object detection, Real-time defect detection},
	pages = {4871--4885},
}

@inproceedings{dosovitskiy_image_2020,
	title = {An {Image} is {Worth} 16x16 {Words}: {Transformers} for {Image} {Recognition} at {Scale}},
	shorttitle = {An {Image} is {Worth} 16x16 {Words}},
	url = {https://openreview.net/forum?id=YicbFdNTTy},
	abstract = {While the Transformer architecture has become the de-facto standard for natural language processing tasks, its applications to computer vision remain limited. In vision, attention is either applied in conjunction with convolutional networks, or used to replace certain components of convolutional networks while keeping their overall structure in place. We show that this reliance on CNNs is not necessary and a pure transformer applied directly to sequences of image patches can perform very well on image classification tasks. When pre-trained on large amounts of data and transferred to multiple mid-sized or small image recognition benchmarks (ImageNet, CIFAR-100, VTAB, etc.), Vision Transformer (ViT) attains excellent results compared to state-of-the-art convolutional networks while requiring substantially fewer computational resources to train.},
	language = {en},
	urldate = {2026-03-05},
	author = {Dosovitskiy, Alexey and Beyer, Lucas and Kolesnikov, Alexander and Weissenborn, Dirk and Zhai, Xiaohua and Unterthiner, Thomas and Dehghani, Mostafa and Minderer, Matthias and Heigold, Georg and Gelly, Sylvain and Uszkoreit, Jakob and Houlsby, Neil},
	month = oct,
	year = {2020},
}

@misc{arnab_vivit_2021,
	title = {{ViViT}: {A} {Video} {Vision} {Transformer}},
	shorttitle = {{ViViT}},
	url = {http://arxiv.org/abs/2103.15691},
	doi = {10.48550/arXiv.2103.15691},
	abstract = {We present pure-transformer based models for video classification, drawing upon the recent success of such models in image classification. Our model extracts spatio-temporal tokens from the input video, which are then encoded by a series of transformer layers. In order to handle the long sequences of tokens encountered in video, we propose several, efficient variants of our model which factorise the spatial- and temporal-dimensions of the input. Although transformer-based models are known to only be effective when large training datasets are available, we show how we can effectively regularise the model during training and leverage pretrained image models to be able to train on comparatively small datasets. We conduct thorough ablation studies, and achieve state-of-the-art results on multiple video classification benchmarks including Kinetics 400 and 600, Epic Kitchens, Something-Something v2 and Moments in Time, outperforming prior methods based on deep 3D convolutional networks. To facilitate further research, we release code at https://github.com/google-research/scenic/tree/main/scenic/projects/vivit},
	urldate = {2026-03-04},
	publisher = {arXiv},
	author = {Arnab, Anurag and Dehghani, Mostafa and Heigold, Georg and Sun, Chen and Lučić, Mario and Schmid, Cordelia},
	month = nov,
	year = {2021},
	note = {arXiv:2103.15691 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition},
}

@misc{yu_point-bert_2022,
	title = {Point-{BERT}: {Pre}-training {3D} {Point} {Cloud} {Transformers} with {Masked} {Point} {Modeling}},
	shorttitle = {Point-{BERT}},
	url = {http://arxiv.org/abs/2111.14819},
	doi = {10.48550/arXiv.2111.14819},
	abstract = {We present Point-BERT, a new paradigm for learning Transformers to generalize the concept of BERT to 3D point cloud. Inspired by BERT, we devise a Masked Point Modeling (MPM) task to pre-train point cloud Transformers. Specifically, we first divide a point cloud into several local point patches, and a point cloud Tokenizer with a discrete Variational AutoEncoder (dVAE) is designed to generate discrete point tokens containing meaningful local information. Then, we randomly mask out some patches of input point clouds and feed them into the backbone Transformers. The pre-training objective is to recover the original point tokens at the masked locations under the supervision of point tokens obtained by the Tokenizer. Extensive experiments demonstrate that the proposed BERT-style pre-training strategy significantly improves the performance of standard point cloud Transformers. Equipped with our pre-training strategy, we show that a pure Transformer architecture attains 93.8\% accuracy on ModelNet40 and 83.1\% accuracy on the hardest setting of ScanObjectNN, surpassing carefully designed point cloud models with much fewer hand-made designs. We also demonstrate that the representations learned by Point-BERT transfer well to new tasks and domains, where our models largely advance the state-of-the-art of few-shot point cloud classification task. The code and pre-trained models are available at https://github.com/lulutang0608/Point-BERT},
	urldate = {2026-03-05},
	publisher = {arXiv},
	author = {Yu, Xumin and Tang, Lulu and Rao, Yongming and Huang, Tiejun and Zhou, Jie and Lu, Jiwen},
	month = jun,
	year = {2022},
	note = {arXiv:2111.14819 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning},
}

@article{jadhav_llm-3d_2025,
	title = {{LLM}-{3D} print: {Large} language models to monitor and control {3D} printing},
	issn = {2214-8604},
	shorttitle = {{LLM}-{3D} print},
	url = {https://www.sciencedirect.com/science/article/pii/S2214860425003926},
	doi = {10.1016/j.addma.2025.105027},
	abstract = {Industry 4.0 has revolutionized manufacturing by driving digitization and shifting the paradigm toward additive manufacturing (AM). Material extrusion (MEX), a core AM method, produces customized and cost-effective products with minimal waste, challenging traditional subtractive manufacturing. Despite its advantages, MEX remains susceptible to defects that can compromise part quality and function, often requiring expert intervention. Existing rule-based and machine learning approaches struggle to generalize across different printers and sensors, while deep learning methods depend on large labeled datasets, limiting their scalability and adaptability. To address these challenges, we introduce a process monitoring and control framework that employs Large Language Models (LLMs) as autonomous controllers for additive manufacturing. Unlike rule-based or heavily data-dependent approaches, our method requires no domain-specific fine-tuning or training. Instead, the LLM leverages in-context learning, self-prompting, and iterative prompt-reason refinement to evaluate print quality from sequential image captures, detect and classify emerging failure modes, and query and modify the printer for relevant operating parameters. Through this adaptive reasoning process, the LLM not only interprets defects but also improves its own decision-making logic, autonomously formulating and executing corrective actions. This demonstrates a rule-free, self-improving approach to process control that extends beyond traditional quality assurance methods. We validated the effectiveness of the proposed framework by comparing it with a control group of engineers with different levels of AM expertise. The evaluation showed that LLM-based agents not only reliably identified common 3D printing errors such as inconsistent extrusion, stringing, warping, and poor layer adhesion, but also determined their causes and corrected them without human intervention. In addition to matching expert-level accuracy, the LLM was able to recognize emerging print errors earlier than human experts, highlighting its value as a proactive controller. To further demonstrate generalizability, we deployed and tested the framework on two different 3D printers with distinct sensor setups, confirming its adaptability across hardware. We also performed compression tests on baseline prints and on prints optimized by the LLM, with the optimized parts showing clear improvements in mechanical performance.},
	urldate = {2025-11-29},
	journal = {Additive Manufacturing},
	author = {Jadhav, Yayati and Pak, Peter and Farimani, Amir Barati},
	month = nov,
	year = {2025},
	keywords = {Generative AI, LLM agents, Machine learning, Process monitoring and correction},
	pages = {105027},
}

@article{liu_visual_2023,
	title = {Visual {Instruction} {Tuning}},
	volume = {36},
	url = {https://papers.nips.cc/paper_files/paper/2023/hash/6dcf277ea32ce3288914faf369fe6de0-Abstract-Conference.html},
	language = {en},
	urldate = {2026-03-03},
	journal = {Advances in Neural Information Processing Systems},
	author = {Liu, Haotian and Li, Chunyuan and Wu, Qingyang and Lee, Yong Jae},
	month = dec,
	year = {2023},
	pages = {34892--34916},
}

@article{french_catastrophic_1999,
	title = {Catastrophic forgetting in connectionist networks},
	volume = {3},
	issn = {1364-6613},
	url = {https://www.sciencedirect.com/science/article/pii/S1364661399012942},
	doi = {10.1016/S1364-6613(99)01294-2},
	abstract = {All natural cognitive systems, and, in particular, our own, gradually forget previously learned information. Plausible models of human cognition should therefore exhibit similar patterns of gradual forgetting of old information as new information is acquired. Only rarely does new learning in natural cognitive systems completely disrupt or erase previously learned information; that is, natural cognitive systems do not, in general, forget ‘catastrophically’. Unfortunately, though, catastrophic forgetting does occur under certain circumstances in distributed connectionist networks. The very features that give these networks their remarkable abilities to generalize, to function in the presence of degraded input, and so on, are found to be the root cause of catastrophic forgetting. The challenge in this field is to discover how to keep the advantages of distributed connectionist networks while avoiding the problem of catastrophic forgetting. In this article the causes, consequences and numerous solutions to the problem of catastrophic forgetting in neural networks are examined. The review will consider how the brain might have overcome this problem and will also explore the consequences of this solution for distributed connectionist networks.},
	number = {4},
	urldate = {2026-02-23},
	journal = {Trends in Cognitive Sciences},
	author = {French, Robert M.},
	month = apr,
	year = {1999},
	keywords = {Catastrophic forgetting, Connectionism, Connectionist networks, Interference, Learning, Memory},
	pages = {128--135},
}

@inproceedings{scialom_fine-tuned_2022,
	address = {Abu Dhabi, United Arab Emirates},
	title = {Fine-tuned {Language} {Models} are {Continual} {Learners}},
	url = {https://aclanthology.org/2022.emnlp-main.410/},
	doi = {10.18653/v1/2022.emnlp-main.410},
	abstract = {Recent work on large language models relies on the intuition that most natural language processing tasks can be described via natural language instructions and that models trained on these instructions show strong zero-shot performance on several standard datasets. However, these models even though impressive still perform poorly on a wide range of tasks outside of their respective training and evaluation sets.To address this limitation, we argue that a model should be able to keep extending its knowledge and abilities, without forgetting previous skills. In spite of the limited success of Continual Learning, we show that Fine-tuned Language Models can be continual learners.We empirically investigate the reason for this success and conclude that Continual Learning emerges from self-supervision pre-training. Our resulting model Continual-T0 (CT0) is able to learn 8 new diverse language generation tasks, while still maintaining good performance on previous tasks, spanning in total of 70 datasets. Finally, we show that CT0 is able to combine instructions in ways it was never trained for, demonstrating some level of instruction compositionality.},
	urldate = {2026-02-23},
	booktitle = {Proceedings of the 2022 {Conference} on {Empirical} {Methods} in {Natural} {Language} {Processing}},
	publisher = {Association for Computational Linguistics},
	author = {Scialom, Thomas and Chakrabarty, Tuhin and Muresan, Smaranda},
	editor = {Goldberg, Yoav and Kozareva, Zornitsa and Zhang, Yue},
	month = dec,
	year = {2022},
	pages = {6107--6122},
}

@misc{chandrasekhar_amgpt_2024,
	title = {{AMGPT}: a {Large} {Language} {Model} for {Contextual} {Querying} in {Additive} {Manufacturing}},
	shorttitle = {{AMGPT}},
	url = {http://arxiv.org/abs/2406.00031},
	doi = {10.48550/arXiv.2406.00031},
	abstract = {Generalized large language models (LLMs) such as GPT-4 may not provide specific answers to queries formulated by materials science researchers. These models may produce a high-level outline but lack the capacity to return detailed instructions on manufacturing and material properties of novel alloys. Enhancing a smaller model with specialized domain knowledge may provide an advantage over large language models which cannot be retrained quickly enough to keep up with the rapid pace of research in metal additive manufacturing (AM). We introduce "AMGPT," a specialized LLM text generator designed for metal AM queries. The goal of AMGPT is to assist researchers and users in navigating the extensive corpus of literature in AM. Instead of training from scratch, we employ a pre-trained Llama2-7B model from Hugging Face in a Retrieval-Augmented Generation (RAG) setup, utilizing it to dynamically incorporate information from \${\textbackslash}sim\$50 AM papers and textbooks in PDF format. Mathpix is used to convert these PDF documents into TeX format, facilitating their integration into the RAG pipeline managed by LlamaIndex. Expert evaluations of this project highlight that specific embeddings from the RAG setup accelerate response times and maintain coherence in the generated text.},
	urldate = {2024-12-01},
	publisher = {arXiv},
	author = {Chandrasekhar, Achuth and Chan, Jonathan and Ogoke, Francis and Ajenifujah, Olabode and Farimani, Amir Barati},
	month = may,
	year = {2024},
	note = {arXiv:2406.00031},
	keywords = {Computer Science - Computation and Language, Computer Science - Machine Learning},
}

@article{luhn_automatic_1958,
	title = {The {Automatic} {Creation} of {Literature} {Abstracts}},
	volume = {2},
	issn = {0018-8646, 0018-8646},
	url = {http://ieeexplore.ieee.org/document/5392672/},
	doi = {10.1147/rd.22.0159},
	abstract = {Excerpts of technical papers and magazine articles that serve the purposes of conventional abstracts have been created entirely by automatic means. In the exploratory research described, the complete text of an article in machine-readable form is scanned by an IBM 704 data-processing machine and analyzed in accordance with a standard program. Statistical information derived from word frequency and distribution is used by the machine to compute a relative measure of significance, first for individual words and then for sentences. Sentences scoring highest in significance are extracted and printed out to become the "auto-abstract."},
	number = {2},
	urldate = {2026-02-14},
	journal = {IBM Journal of Research and Development},
	author = {Luhn, H. P.},
	month = apr,
	year = {1958},
	pages = {159--165},
}

@techreport{buckley_implementation_1985,
	address = {USA},
	type = {Technical {Report}},
	title = {Implementation of the {SMART} {Information} {Retrieval} {System}},
	abstract = {ABSTRACT NOT SUPPLIED},
	institution = {Cornell University},
	author = {Buckley, Chris},
	month = apr,
	year = {1985},
}

@misc{wang_diversity_2025,
	title = {Diversity {Enhances} an {LLM}'s {Performance} in {RAG} and {Long}-context {Task}},
	url = {http://arxiv.org/abs/2502.09017},
	doi = {10.48550/arXiv.2502.09017},
	abstract = {The rapid advancements in large language models (LLMs) have highlighted the challenge of context window limitations, primarily due to the quadratic time complexity of the self-attention mechanism ({\textbackslash}(O(N{\textasciicircum}2){\textbackslash}), where {\textbackslash}(N{\textbackslash}) denotes the context window length). This constraint impacts tasks such as retrieval-augmented generation (RAG) in question answering (Q{\textbackslash}\&A) and long context summarization. A common approach involves selecting content with the highest similarity to the query; however, this often leads to redundancy and the exclusion of diverse yet relevant information. Building on principles from Maximal Marginal Relevance (MMR) and Farthest Point Sampling (FPS), we integrate diversity into the content selection process. Our findings reveal that incorporating diversity substantially increases the recall of selecting relevant sentences or chunks before LLM-based Q{\textbackslash}\&A and summarization. These results highlight the importance of maintaining diversity in future LLM applications to further improve summarization and Q{\textbackslash}\&A outcomes.},
	urldate = {2026-02-14},
	publisher = {arXiv},
	author = {Wang, Zhichao and Bi, Bin and Luo, Yanqi and Asur, Sitaram and Cheng, Claire Na},
	month = apr,
	year = {2025},
	note = {arXiv:2502.09017 [cs]},
	keywords = {Computer Science - Computation and Language, Computer Science - Machine Learning},
}

@book{salton_automatic_1988,
	address = {USA},
	title = {Automatic text processing},
	publisher = {Addison-Wesley Longman Publishing Co., Inc.},
	author = {Salton, Gerald},
	month = mar,
	year = {1988},
}

@inproceedings{kupiec_trainable_1995,
	address = {Seattle, Washington, United States},
	title = {A trainable document summarizer},
	copyright = {https://www.acm.org/publications/policies/copyright\_policy\#Background},
	isbn = {978-0-89791-714-8},
	url = {http://portal.acm.org/citation.cfm?doid=215206.215333},
	doi = {10.1145/215206.215333},
	abstract = {●● ● ● ● To summarize is to reduce in complexity, and hence in length, while retaining some of the essential qualities of the original. This paper focusses on document extracts, a particular kind of computed document summary. Document extracts consisting of roughly 20\% of the original cart be as informative as the full text of a document, which suggests that even shorter extracts may be useful indicative summmies. The trends in our results are in agreement with those of Edmundson who used a subjectively weighted combination of features as opposed to training the feature weights using a corpus.},
	language = {en},
	urldate = {2026-02-14},
	booktitle = {Proceedings of the 18th annual international {ACM} {SIGIR} conference on {Research} and development in information retrieval  - {SIGIR} '95},
	publisher = {ACM Press},
	author = {Kupiec, Julian and Pedersen, Jan and Chen, Francine},
	year = {1995},
	pages = {68--73},
}

@inproceedings{goldstein_summarization_1998,
	address = {USA},
	series = {{TIPSTER} '98},
	title = {Summarization: (1) using {MMR} for diversity - based reranking and (2) evaluating summaries},
	shorttitle = {Summarization},
	url = {https://dl.acm.org/doi/10.3115/1119089.1119120},
	doi = {10.3115/1119089.1119120},
	abstract = {This paper develops a method for combining query-relevance with information-novelty in the context of text retrieval and summarization. The Maximal Marginal Relevance (MMR) criterion strives to reduce redundancy while maintaining query relevance in reranking retrieved documents and in selecting appropriate passages for text summarization. Preliminary results indicate some benefits for MMR diversity ranking in ad-hoc query and in single document summarization. The latter are borne out by the trial-run (unofficial) TREC-style evaluation of summarization systems. However, the clearest advantage is demonstrated in the automated construction of large document and non-redundant multi-document summaries, where MMR results are clearly superior to non-MMR passage selection. This paper also discusses our preliminary evaluation of summarization methods for single documents.},
	urldate = {2026-02-14},
	booktitle = {Proceedings of a workshop on held at {Baltimore}, {Maryland}: {October} 13-15, 1998},
	publisher = {Association for Computational Linguistics},
	author = {Goldstein, Jade and Carbonell, Jaime},
	month = oct,
	year = {1998},
	pages = {181--195},
}

@misc{li_long_2024,
	title = {Long {Context} vs. {RAG} for {LLMs}: {An} {Evaluation} and {Revisits}},
	shorttitle = {Long {Context} vs. {RAG} for {LLMs}},
	url = {http://arxiv.org/abs/2501.01880},
	doi = {10.48550/arXiv.2501.01880},
	abstract = {Extending context windows (i.e., Long Context, LC) and using retrievers to selectively access relevant information (i.e., Retrieval-Augmented Generation, RAG) are the two main strategies to enable LLMs to incorporate extremely long external contexts. This paper revisits recent studies on this topic, highlighting their key insights and discrepancies. We then provide a more comprehensive evaluation by filtering out questions answerable without external context, identifying the most effective retrieval methods, and expanding the datasets. We show that LC generally outperforms RAG in question-answering benchmarks, especially for Wikipedia-based questions. Summarization-based retrieval performs comparably to LC, while chunk-based retrieval lags behind. However, RAG has advantages in dialogue-based and general question queries. These insights underscore the trade-offs between RAG and LC strategies, offering guidance for future optimization of LLMs with external knowledge sources. We also provide an in-depth discussion on this topic, highlighting the overlooked importance of context relevance in existing studies.},
	urldate = {2026-02-11},
	publisher = {arXiv},
	author = {Li, Xinze and Cao, Yixin and Ma, Yubo and Sun, Aixin},
	month = dec,
	year = {2024},
	note = {arXiv:2501.01880 [cs]},
	keywords = {Computer Science - Computation and Language},
}

@misc{packer_memgpt_2024,
	title = {{MemGPT}: {Towards} {LLMs} as {Operating} {Systems}},
	shorttitle = {{MemGPT}},
	url = {http://arxiv.org/abs/2310.08560},
	doi = {10.48550/arXiv.2310.08560},
	abstract = {Large language models (LLMs) have revolutionized AI, but are constrained by limited context windows, hindering their utility in tasks like extended conversations and document analysis. To enable using context beyond limited context windows, we propose virtual context management, a technique drawing inspiration from hierarchical memory systems in traditional operating systems that provide the appearance of large memory resources through data movement between fast and slow memory. Using this technique, we introduce MemGPT (Memory-GPT), a system that intelligently manages different memory tiers in order to effectively provide extended context within the LLM's limited context window, and utilizes interrupts to manage control flow between itself and the user. We evaluate our OS-inspired design in two domains where the limited context windows of modern LLMs severely handicaps their performance: document analysis, where MemGPT is able to analyze large documents that far exceed the underlying LLM's context window, and multi-session chat, where MemGPT can create conversational agents that remember, reflect, and evolve dynamically through long-term interactions with their users. We release MemGPT code and data for our experiments at https://memgpt.ai.},
	urldate = {2026-02-11},
	publisher = {arXiv},
	author = {Packer, Charles and Wooders, Sarah and Lin, Kevin and Fang, Vivian and Patil, Shishir G. and Stoica, Ion and Gonzalez, Joseph E.},
	month = feb,
	year = {2024},
	note = {arXiv:2310.08560 [cs]},
	keywords = {Computer Science - Artificial Intelligence},
}

@misc{lewis_bart_2019,
	title = {{BART}: {Denoising} {Sequence}-to-{Sequence} {Pre}-training for {Natural} {Language} {Generation}, {Translation}, and {Comprehension}},
	shorttitle = {{BART}},
	url = {http://arxiv.org/abs/1910.13461},
	doi = {10.48550/arXiv.1910.13461},
	abstract = {We present BART, a denoising autoencoder for pretraining sequence-to-sequence models. BART is trained by (1) corrupting text with an arbitrary noising function, and (2) learning a model to reconstruct the original text. It uses a standard Tranformer-based neural machine translation architecture which, despite its simplicity, can be seen as generalizing BERT (due to the bidirectional encoder), GPT (with the left-to-right decoder), and many other more recent pretraining schemes. We evaluate a number of noising approaches, finding the best performance by both randomly shuffling the order of the original sentences and using a novel in-filling scheme, where spans of text are replaced with a single mask token. BART is particularly effective when fine tuned for text generation but also works well for comprehension tasks. It matches the performance of RoBERTa with comparable training resources on GLUE and SQuAD, achieves new state-of-the-art results on a range of abstractive dialogue, question answering, and summarization tasks, with gains of up to 6 ROUGE. BART also provides a 1.1 BLEU increase over a back-translation system for machine translation, with only target language pretraining. We also report ablation experiments that replicate other pretraining schemes within the BART framework, to better measure which factors most influence end-task performance.},
	urldate = {2026-02-11},
	publisher = {arXiv},
	author = {Lewis, Mike and Liu, Yinhan and Goyal, Naman and Ghazvininejad, Marjan and Mohamed, Abdelrahman and Levy, Omer and Stoyanov, Ves and Zettlemoyer, Luke},
	month = oct,
	year = {2019},
	note = {arXiv:1910.13461 [cs]},
	keywords = {Computer Science - Computation and Language, Computer Science - Machine Learning, Statistics - Machine Learning},
}

@inproceedings{lewis_retrieval-augmented_2020,
	address = {Red Hook, NY, USA},
	series = {{NIPS} '20},
	title = {Retrieval-augmented generation for knowledge-intensive {NLP} tasks},
	isbn = {978-1-7138-2954-6},
	url = {https://dl.acm.org/doi/10.5555/3495724.3496517},
	abstract = {Large pre-trained language models have been shown to store factual knowledge in their parameters, and achieve state-of-the-art results when fine-tuned on downstream NLP tasks. However, their ability to access and precisely manipulate knowledge is still limited, and hence on knowledge-intensive tasks, their performance lags behind task-specific architectures. Additionally, providing provenance for their decisions and updating their world knowledge remain open research problems. Pre-trained models with a differentiable access mechanism to explicit non-parametric memory can overcome this issue, but have so far been only investigated for extractive downstream tasks. We explore a general-purpose fine-tuning recipe for retrieval-augmented generation (RAG) — models which combine pre-trained parametric and non-parametric memory for language generation. We introduce RAG models where the parametric memory is a pre-trained seq2seq model and the non-parametric memory is a dense vector index of Wikipedia, accessed with a pre-trained neural retriever. We compare two RAG formulations, one which conditions on the same retrieved passages across the whole generated sequence, and another which can use different passages per token. We fine-tune and evaluate our models on a wide range of knowledge-intensive NLP tasks and set the state of the art on three open domain QA tasks, outperforming parametric seq2seq models and task-specific retrieve-and-extract architectures. For language generation tasks, we find that RAG models generate more specific, diverse and factual language than a state-of-the-art parametric-only seq2seq baseline.},
	urldate = {2026-02-10},
	booktitle = {Proceedings of the 34th {International} {Conference} on {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates Inc.},
	author = {Lewis, Patrick and Perez, Ethan and Piktus, Aleksandra and Petroni, Fabio and Karpukhin, Vladimir and Goyal, Naman and Küttler, Heinrich and Lewis, Mike and Yih, Wen-tau and Rocktäschel, Tim and Riedel, Sebastian and Kiela, Douwe},
	month = dec,
	year = {2020},
	pages = {9459--9474},
}

@inproceedings{karpukhin_dense_2020,
	address = {Online},
	title = {Dense {Passage} {Retrieval} for {Open}-{Domain} {Question} {Answering}},
	url = {https://aclanthology.org/2020.emnlp-main.550/},
	doi = {10.18653/v1/2020.emnlp-main.550},
	abstract = {Open-domain question answering relies on efficient passage retrieval to select candidate contexts, where traditional sparse vector space models, such as TF-IDF or BM25, are the de facto method. In this work, we show that retrieval can be practically implemented using dense representations alone, where embeddings are learned from a small number of questions and passages by a simple dual-encoder framework. When evaluated on a wide range of open-domain QA datasets, our dense retriever outperforms a strong Lucene-BM25 system greatly by 9\%-19\% absolute in terms of top-20 passage retrieval accuracy, and helps our end-to-end QA system establish new state-of-the-art on multiple open-domain QA benchmarks.},
	urldate = {2026-02-10},
	booktitle = {Proceedings of the 2020 {Conference} on {Empirical} {Methods} in {Natural} {Language} {Processing} ({EMNLP})},
	publisher = {Association for Computational Linguistics},
	author = {Karpukhin, Vladimir and Oguz, Barlas and Min, Sewon and Lewis, Patrick and Wu, Ledell and Edunov, Sergey and Chen, Danqi and Yih, Wen-tau},
	editor = {Webber, Bonnie and Cohn, Trevor and He, Yulan and Liu, Yang},
	month = nov,
	year = {2020},
	pages = {6769--6781},
}

@article{carbonell_use_1997,
	title = {The {Use} of {MMR} and {Diversity}-{Based} {Reranking} in {Document} {Reranking} and {Summarization}},
	url = {https://kilthub.cmu.edu/articles/journal_contribution/The_Use_of_MMR_and_Diversity-Based_Reranking_in_Document_Reranking_and_Summarization/6610814/1},
	doi = {10.1184/R1/6610814.v1},
	abstract = {This paper presents a method for combining query-relevance with information-novelty in the context of text retrieval and summarization. The Maximal Marginal Relevance (MMR) criterion strives to reduce redundancy while maintaining query relevance in re-ranking retrieved documents and in selecting appropriate passages for text summarization. Preliminary results indicate some benefits for MMR diversity ranking in document retrieval and in single document summarization. The latter are borne out by the recent results of the SUMMAC conference in the evaluation of summarization systems. However, the clearest advantage is demonstrated in constructing non-redundant multi-document summaries, where MMR results are clearly superior to non-MMR passage selection.},
	language = {en},
	urldate = {2026-02-11},
	publisher = {Carnegie Mellon University},
	author = {Carbonell, Jaime G. and Goldstein, Jade},
	month = jul,
	year = {1997},
}

@misc{bai_training_2022,
	title = {Training a {Helpful} and {Harmless} {Assistant} with {Reinforcement} {Learning} from {Human} {Feedback}},
	url = {http://arxiv.org/abs/2204.05862},
	doi = {10.48550/arXiv.2204.05862},
	abstract = {We apply preference modeling and reinforcement learning from human feedback (RLHF) to finetune language models to act as helpful and harmless assistants. We find this alignment training improves performance on almost all NLP evaluations, and is fully compatible with training for specialized skills such as python coding and summarization. We explore an iterated online mode of training, where preference models and RL policies are updated on a weekly cadence with fresh human feedback data, efficiently improving our datasets and models. Finally, we investigate the robustness of RLHF training, and identify a roughly linear relation between the RL reward and the square root of the KL divergence between the policy and its initialization. Alongside our main results, we perform peripheral analyses on calibration, competing objectives, and the use of OOD detection, compare our models with human writers, and provide samples from our models using prompts appearing in recent related work.},
	urldate = {2026-02-03},
	publisher = {arXiv},
	author = {Bai, Yuntao and Jones, Andy and Ndousse, Kamal and Askell, Amanda and Chen, Anna and DasSarma, Nova and Drain, Dawn and Fort, Stanislav and Ganguli, Deep and Henighan, Tom and Joseph, Nicholas and Kadavath, Saurav and Kernion, Jackson and Conerly, Tom and El-Showk, Sheer and Elhage, Nelson and Hatfield-Dodds, Zac and Hernandez, Danny and Hume, Tristan and Johnston, Scott and Kravec, Shauna and Lovitt, Liane and Nanda, Neel and Olsson, Catherine and Amodei, Dario and Brown, Tom and Clark, Jack and McCandlish, Sam and Olah, Chris and Mann, Ben and Kaplan, Jared},
	month = apr,
	year = {2022},
	note = {arXiv:2204.05862 [cs]},
	keywords = {Computer Science - Computation and Language, Computer Science - Machine Learning},
}

@inproceedings{yao_react_2022,
	title = {{ReAct}: {Synergizing} {Reasoning} and {Acting} in {Language} {Models}},
	shorttitle = {{ReAct}},
	url = {https://openreview.net/forum?id=WE_vluYUL-X},
	abstract = {While large language models (LLMs) have demonstrated impressive capabilities across tasks in language understanding and interactive decision making, their abilities for reasoning (e.g. chain-of-thought prompting) and acting (e.g. action plan generation) have primarily been studied as separate topics. In this paper, we explore the use of LLMs to generate both reasoning traces and task-specific actions in an interleaved manner, allowing for greater synergy between the two: reasoning traces help the model induce, track, and update action plans as well as handle exceptions, while actions allow it to interface with external sources, such as knowledge bases or environments, to gather additional information. We apply our approach, named ReAct, to a diverse set of language and decision making tasks and demonstrate its effectiveness over state-of-the-art baselines, as well as improved human interpretability and trustworthiness over methods without reasoning or acting components. Concretely, on question answering (HotpotQA) and fact verification (Fever), ReAct overcomes issues of hallucination and error propagation prevalent in chain-of-thought reasoning by interacting with a simple Wikipedia API, and generates human-like task-solving trajectories that are more interpretable than baselines without reasoning traces. On two interactive decision making benchmarks (ALFWorld and WebShop), ReAct outperforms imitation and reinforcement learning methods by an absolute success rate of 34\% and 10\% respectively, while being prompted with only one or two in-context examples.},
	language = {en},
	urldate = {2026-01-23},
	author = {Yao, Shunyu and Zhao, Jeffrey and Yu, Dian and Du, Nan and Shafran, Izhak and Narasimhan, Karthik R. and Cao, Yuan},
	month = sep,
	year = {2022},
}

@misc{openai_gpt-oss-120b_2025,
	title = {gpt-oss-120b \& gpt-oss-20b {Model} {Card}},
	url = {http://arxiv.org/abs/2508.10925},
	doi = {10.48550/arXiv.2508.10925},
	abstract = {We present gpt-oss-120b and gpt-oss-20b, two open-weight reasoning models that push the frontier of accuracy and inference cost. The models use an efficient mixture-of-expert transformer architecture and are trained using large-scale distillation and reinforcement learning. We optimize the models to have strong agentic capabilities (deep research browsing, python tool use, and support for developer-provided functions), all while using a rendered chat format that enables clear instruction following and role delineation. Both models achieve strong results on benchmarks ranging from mathematics, coding, and safety. We release the model weights, inference implementations, tool environments, and tokenizers under an Apache 2.0 license to enable broad use and further research.},
	urldate = {2026-01-25},
	publisher = {arXiv},
	author = {OpenAI and Agarwal, Sandhini and Ahmad, Lama and Ai, Jason and Altman, Sam and Applebaum, Andy and Arbus, Edwin and Arora, Rahul K. and Bai, Yu and Baker, Bowen and Bao, Haiming and Barak, Boaz and Bennett, Ally and Bertao, Tyler and Brett, Nivedita and Brevdo, Eugene and Brockman, Greg and Bubeck, Sebastien and Chang, Che and Chen, Kai and Chen, Mark and Cheung, Enoch and Clark, Aidan and Cook, Dan and Dukhan, Marat and Dvorak, Casey and Fives, Kevin and Fomenko, Vlad and Garipov, Timur and Georgiev, Kristian and Glaese, Mia and Gogineni, Tarun and Goucher, Adam and Gross, Lukas and Guzman, Katia Gil and Hallman, John and Hehir, Jackie and Heidecke, Johannes and Helyar, Alec and Hu, Haitang and Huet, Romain and Huh, Jacob and Jain, Saachi and Johnson, Zach and Koch, Chris and Kofman, Irina and Kundel, Dominik and Kwon, Jason and Kyrylov, Volodymyr and Le, Elaine Ya and Leclerc, Guillaume and Lennon, James Park and Lessans, Scott and Lezcano-Casado, Mario and Li, Yuanzhi and Li, Zhuohan and Lin, Ji and Liss, Jordan and Lily and Liu and Liu, Jiancheng and Lu, Kevin and Lu, Chris and Martinovic, Zoran and McCallum, Lindsay and McGrath, Josh and McKinney, Scott and McLaughlin, Aidan and Mei, Song and Mostovoy, Steve and Mu, Tong and Myles, Gideon and Neitz, Alexander and Nichol, Alex and Pachocki, Jakub and Paino, Alex and Palmie, Dana and Pantuliano, Ashley and Parascandolo, Giambattista and Park, Jongsoo and Pathak, Leher and Paz, Carolina and Peran, Ludovic and Pimenov, Dmitry and Pokrass, Michelle and Proehl, Elizabeth and Qiu, Huida and Raila, Gaby and Raso, Filippo and Ren, Hongyu and Richardson, Kimmy and Robinson, David and Rotsted, Bob and Salman, Hadi and Sanjeev, Suvansh and Schwarzer, Max and Sculley, D. and Sikchi, Harshit and Simon, Kendal and Singhal, Karan and Song, Yang and Stuckey, Dane and Sun, Zhiqing and Tillet, Philippe and Toizer, Sam and Tsimpourlas, Foivos and Vyas, Nikhil and Wallace, Eric and Wang, Xin and Wang, Miles and Watkins, Olivia and Weil, Kevin and Wendling, Amy and Whinnery, Kevin and Whitney, Cedric and Wong, Hannah and Yang, Lin and Yang, Yu and Yasunaga, Michihiro and Ying, Kristen and Zaremba, Wojciech and Zhan, Wenting and Zhang, Cyril and Zhang, Brian and Zhang, Eddie and Zhao, Shengjia},
	month = aug,
	year = {2025},
	note = {arXiv:2508.10925 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computation and Language},
}

@misc{baker_monitoring_2025,
	title = {Monitoring {Reasoning} {Models} for {Misbehavior} and the {Risks} of {Promoting} {Obfuscation}},
	url = {http://arxiv.org/abs/2503.11926},
	doi = {10.48550/arXiv.2503.11926},
	abstract = {Mitigating reward hacking--where AI systems misbehave due to flaws or misspecifications in their learning objectives--remains a key challenge in constructing capable and aligned models. We show that we can monitor a frontier reasoning model, such as OpenAI o3-mini, for reward hacking in agentic coding environments by using another LLM that observes the model's chain-of-thought (CoT) reasoning. CoT monitoring can be far more effective than monitoring agent actions and outputs alone, and we further found that a LLM weaker than o3-mini, namely GPT-4o, can effectively monitor a stronger model. Because CoT monitors can be effective at detecting exploits, it is natural to ask whether those exploits can be suppressed by incorporating a CoT monitor directly into the agent's training objective. While we show that integrating CoT monitors into the reinforcement learning reward can indeed produce more capable and more aligned agents in the low optimization regime, we find that with too much optimization, agents learn obfuscated reward hacking, hiding their intent within the CoT while still exhibiting a significant rate of reward hacking. Because it is difficult to tell when CoTs have become obfuscated, it may be necessary to pay a monitorability tax by not applying strong optimization pressures directly to the chain-of-thought, ensuring that CoTs remain monitorable and useful for detecting misaligned behavior.},
	urldate = {2026-01-25},
	publisher = {arXiv},
	author = {Baker, Bowen and Huizinga, Joost and Gao, Leo and Dou, Zehao and Guan, Melody Y. and Madry, Aleksander and Zaremba, Wojciech and Pachocki, Jakub and Farhi, David},
	month = mar,
	year = {2025},
	note = {arXiv:2503.11926 [cs]},
	keywords = {Computer Science - Artificial Intelligence},
}

@misc{noauthor_langchain_nodate,
	title = {{LangChain} overview},
	url = {https://docs.langchain.com/oss/python/langchain/overview},
	abstract = {LangChain is an open source framework with a pre-built agent architecture and integrations for any model or tool — so you can build agents that adapt as fast as the ecosystem evolves},
	language = {en},
	urldate = {2026-01-22},
	journal = {Docs by LangChain},
}

@misc{noauthor_crewai_nodate,
	title = {{CrewAI} {Documentation} - {CrewAI}},
	url = {https://docs.crewai.com},
	abstract = {Build collaborative AI agents, crews, and flows — production ready from day one.},
	language = {en},
	urldate = {2026-01-22},
	journal = {CrewAI Documentation},
}

@misc{noauthor_openaiopenai-cookbook_nodate,
	title = {openai/openai-cookbook: {Examples} and guides for using the {OpenAI} {API}},
	url = {https://github.com/openai/openai-cookbook},
	urldate = {2026-01-22},
}

@inproceedings{wu_autogen_2024,
	title = {{AutoGen}: {Enabling} {Next}-{Gen} {LLM} {Applications} via {Multi}-{Agent} {Conversation}},
	url = {https://www.microsoft.com/en-us/research/publication/autogen-enabling-next-gen-llm-applications-via-multi-agent-conversation-framework/},
	abstract = {We present AutoGen, an open-source framework that allows developers to build LLM applications by composing multiple agents to converse with each other to accomplish tasks. AutoGen agents are customizable, conversable, and can operate in various modes that employ combinations of LLMs, human inputs, and tools. It also enables developers to create flexible agent behaviors and conversation patterns for different applications using both natural language and code. AutoGen serves as a generic infrastructure and is widely used by AI practitioners and researchers to build diverse applications of various complexities and LLM capacities. We demonstrate the framework’s effectiveness with several pilot applications, on domains ranging from mathematics and coding to question-answering, supply-chain optimization, online decision-making, and entertainment.},
	booktitle = {{COLM} 2024},
	author = {Wu, Qingyun and Bansal, Gagan and Zhang, Jieyu and Wu, Yiran and Li, Beibin and Zhu, Erkang (Eric) and Jiang, Li and Zhang, Xiaoyun and Zhang, Shaokun and Awadallah, Ahmed and White, Ryen W. and Burger, Doug and Wang, Chi},
	month = aug,
	year = {2024},
}

@misc{zelikman_star_2022,
	title = {{STaR}: {Bootstrapping} {Reasoning} {With} {Reasoning}},
	shorttitle = {{STaR}},
	url = {http://arxiv.org/abs/2203.14465},
	doi = {10.48550/arXiv.2203.14465},
	abstract = {Generating step-by-step "chain-of-thought" rationales improves language model performance on complex reasoning tasks like mathematics or commonsense question-answering. However, inducing language model rationale generation currently requires either constructing massive rationale datasets or sacrificing accuracy by using only few-shot inference. We propose a technique to iteratively leverage a small number of rationale examples and a large dataset without rationales, to bootstrap the ability to perform successively more complex reasoning. This technique, the "Self-Taught Reasoner" (STaR), relies on a simple loop: generate rationales to answer many questions, prompted with a few rationale examples; if the generated answers are wrong, try again to generate a rationale given the correct answer; fine-tune on all the rationales that ultimately yielded correct answers; repeat. We show that STaR significantly improves performance on multiple datasets compared to a model fine-tuned to directly predict final answers, and performs comparably to fine-tuning a 30\${\textbackslash}times\$ larger state-of-the-art language model on CommensenseQA. Thus, STaR lets a model improve itself by learning from its own generated reasoning.},
	urldate = {2026-01-22},
	publisher = {arXiv},
	author = {Zelikman, Eric and Wu, Yuhuai and Mu, Jesse and Goodman, Noah D.},
	month = may,
	year = {2022},
	note = {arXiv:2203.14465 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computation and Language, Computer Science - Machine Learning},
}

@misc{wei_chain--thought_2023,
	title = {Chain-of-{Thought} {Prompting} {Elicits} {Reasoning} in {Large} {Language} {Models}},
	url = {http://arxiv.org/abs/2201.11903},
	doi = {10.48550/arXiv.2201.11903},
	abstract = {We explore how generating a chain of thought -- a series of intermediate reasoning steps -- significantly improves the ability of large language models to perform complex reasoning. In particular, we show how such reasoning abilities emerge naturally in sufficiently large language models via a simple method called chain of thought prompting, where a few chain of thought demonstrations are provided as exemplars in prompting. Experiments on three large language models show that chain of thought prompting improves performance on a range of arithmetic, commonsense, and symbolic reasoning tasks. The empirical gains can be striking. For instance, prompting a 540B-parameter language model with just eight chain of thought exemplars achieves state of the art accuracy on the GSM8K benchmark of math word problems, surpassing even finetuned GPT-3 with a verifier.},
	urldate = {2026-01-22},
	publisher = {arXiv},
	author = {Wei, Jason and Wang, Xuezhi and Schuurmans, Dale and Bosma, Maarten and Ichter, Brian and Xia, Fei and Chi, Ed and Le, Quoc and Zhou, Denny},
	month = jan,
	year = {2023},
	note = {arXiv:2201.11903 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computation and Language},
}

@misc{kojima_large_2023,
	title = {Large {Language} {Models} are {Zero}-{Shot} {Reasoners}},
	url = {http://arxiv.org/abs/2205.11916},
	doi = {10.48550/arXiv.2205.11916},
	abstract = {Pretrained large language models (LLMs) are widely used in many sub-fields of natural language processing (NLP) and generally known as excellent few-shot learners with task-specific exemplars. Notably, chain of thought (CoT) prompting, a recent technique for eliciting complex multi-step reasoning through step-by-step answer examples, achieved the state-of-the-art performances in arithmetics and symbolic reasoning, difficult system-2 tasks that do not follow the standard scaling laws for LLMs. While these successes are often attributed to LLMs' ability for few-shot learning, we show that LLMs are decent zero-shot reasoners by simply adding "Let's think step by step" before each answer. Experimental results demonstrate that our Zero-shot-CoT, using the same single prompt template, significantly outperforms zero-shot LLM performances on diverse benchmark reasoning tasks including arithmetics (MultiArith, GSM8K, AQUA-RAT, SVAMP), symbolic reasoning (Last Letter, Coin Flip), and other logical reasoning tasks (Date Understanding, Tracking Shuffled Objects), without any hand-crafted few-shot examples, e.g. increasing the accuracy on MultiArith from 17.7\% to 78.7\% and GSM8K from 10.4\% to 40.7\% with large InstructGPT model (text-davinci-002), as well as similar magnitudes of improvements with another off-the-shelf large model, 540B parameter PaLM. The versatility of this single prompt across very diverse reasoning tasks hints at untapped and understudied fundamental zero-shot capabilities of LLMs, suggesting high-level, multi-task broad cognitive capabilities may be extracted by simple prompting. We hope our work not only serves as the minimal strongest zero-shot baseline for the challenging reasoning benchmarks, but also highlights the importance of carefully exploring and analyzing the enormous zero-shot knowledge hidden inside LLMs before crafting finetuning datasets or few-shot exemplars.},
	urldate = {2026-01-22},
	publisher = {arXiv},
	author = {Kojima, Takeshi and Gu, Shixiang Shane and Reid, Machel and Matsuo, Yutaka and Iwasawa, Yusuke},
	month = jan,
	year = {2023},
	note = {arXiv:2205.11916 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computation and Language, Computer Science - Machine Learning},
}

@misc{zhang_instruction_2025,
	title = {Instruction {Tuning} for {Large} {Language} {Models}: {A} {Survey}},
	shorttitle = {Instruction {Tuning} for {Large} {Language} {Models}},
	url = {http://arxiv.org/abs/2308.10792},
	doi = {10.48550/arXiv.2308.10792},
	abstract = {This paper surveys research works in the quickly advancing field of instruction tuning (IT), which can also be referred to as supervised fine-tuning (SFT){\textbackslash}footnote\{In this paper, unless specified otherwise, supervised fine-tuning (SFT) and instruction tuning (IT) are used interchangeably.\}, a crucial technique to enhance the capabilities and controllability of large language models (LLMs). Instruction tuning refers to the process of further training LLMs on a dataset consisting of {\textbackslash}textsc\{(instruction, output)\} pairs in a supervised fashion, which bridges the gap between the next-word prediction objective of LLMs and the users' objective of having LLMs adhere to human instructions. In this work, we make a systematic review of the literature, including the general methodology of SFT, the construction of SFT datasets, the training of SFT models, and applications to different modalities, domains and application, along with analysis on aspects that influence the outcome of SFT (e.g., generation of instruction outputs, size of the instruction dataset, etc). We also review the potential pitfalls of SFT along with criticism against it, along with efforts pointing out current deficiencies of existing strategies and suggest some avenues for fruitful research. Project Page: github.com/xiaoya-li/Instruction-Tuning-Survey},
	urldate = {2026-01-22},
	publisher = {arXiv},
	author = {Zhang, Shengyu and Dong, Linfeng and Li, Xiaoya and Zhang, Sen and Sun, Xiaofei and Wang, Shuhe and Li, Jiwei and Hu, Runyi and Zhang, Tianwei and Wu, Fei and Wang, Guoyin},
	month = oct,
	year = {2025},
	note = {arXiv:2308.10792 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computation and Language, Computer Science - Machine Learning},
}

@misc{devlin_bert_2019,
	title = {{BERT}: {Pre}-training of {Deep} {Bidirectional} {Transformers} for {Language} {Understanding}},
	shorttitle = {{BERT}},
	url = {http://arxiv.org/abs/1810.04805},
	doi = {10.48550/arXiv.1810.04805},
	abstract = {We introduce a new language representation model called BERT, which stands for Bidirectional Encoder Representations from Transformers. Unlike recent language representation models, BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly conditioning on both left and right context in all layers. As a result, the pre-trained BERT model can be fine-tuned with just one additional output layer to create state-of-the-art models for a wide range of tasks, such as question answering and language inference, without substantial task-specific architecture modifications. BERT is conceptually simple and empirically powerful. It obtains new state-of-the-art results on eleven natural language processing tasks, including pushing the GLUE score to 80.5\% (7.7\% point absolute improvement), MultiNLI accuracy to 86.7\% (4.6\% absolute improvement), SQuAD v1.1 question answering Test F1 to 93.2 (1.5 point absolute improvement) and SQuAD v2.0 Test F1 to 83.1 (5.1 point absolute improvement).},
	urldate = {2026-01-22},
	publisher = {arXiv},
	author = {Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina},
	month = may,
	year = {2019},
	note = {arXiv:1810.04805 [cs]},
	keywords = {Computer Science - Computation and Language},
}

@inproceedings{sennrich_neural_2016,
	address = {Berlin, Germany},
	title = {Neural {Machine} {Translation} of {Rare} {Words} with {Subword} {Units}},
	url = {https://aclanthology.org/P16-1162/},
	doi = {10.18653/v1/P16-1162},
	urldate = {2026-01-21},
	booktitle = {Proceedings of the 54th {Annual} {Meeting} of the {Association} for {Computational} {Linguistics} ({Volume} 1: {Long} {Papers})},
	publisher = {Association for Computational Linguistics},
	author = {Sennrich, Rico and Haddow, Barry and Birch, Alexandra},
	editor = {Erk, Katrin and Smith, Noah A.},
	month = aug,
	year = {2016},
	pages = {1715--1725},
}

@article{gage_new_1994,
	title = {A new algorithm for data compression},
	url = {https://www.semanticscholar.org/paper/A-new-algorithm-for-data-compression-Gage/1aa9c0045f1fe8c79cce03c7c14ef4b4643a21f8},
	abstract = {Data compression is becoming increasingly important as a way to stretch disk space and speed up data transfers. This article describes a simple general-purpose data compression algo-rithm, called Byte Pair Encoding (BPE), which provides almost as much compression as the popular Lempel, Ziv},
	urldate = {2026-01-15},
	journal = {The C Users Journal archive},
	author = {Gage, Philip},
	month = feb,
	year = {1994},
}

@misc{vaswani_attention_2023,
	title = {Attention {Is} {All} {You} {Need}},
	url = {http://arxiv.org/abs/1706.03762},
	doi = {10.48550/arXiv.1706.03762},
	abstract = {The dominant sequence transduction models are based on complex recurrent or convolutional neural networks in an encoder-decoder configuration. The best performing models also connect the encoder and decoder through an attention mechanism. We propose a new simple network architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two machine translation tasks show these models to be superior in quality while being more parallelizable and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task, improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training costs of the best models from the literature. We show that the Transformer generalizes well to other tasks by applying it successfully to English constituency parsing both with large and limited training data.},
	urldate = {2026-01-13},
	publisher = {arXiv},
	author = {Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N. and Kaiser, Lukasz and Polosukhin, Illia},
	month = aug,
	year = {2023},
	note = {arXiv:1706.03762 [cs]},
	keywords = {Computer Science - Computation and Language, Computer Science - Machine Learning},
}

@article{kaplan_scaling_2020,
	title = {Scaling {Laws} for {Neural} {Language} {Models}},
	url = {http://arxiv.org/abs/2001.08361},
	doi = {10.48550/arXiv.2001.08361},
	abstract = {We study empirical scaling laws for language model performance on the cross-entropy loss. The loss scales as a power-law with model size, dataset size, and the amount of compute used for training, with some trends spanning more than seven orders of magnitude. Other architectural details such as network width or depth have minimal effects within a wide range. Simple equations govern the dependence of overfitting on model/dataset size and the dependence of training speed on model size. These relationships allow us to determine the optimal allocation of a fixed compute budget. Larger models are significantly more sample-efficient, such that optimally compute-efficient training involves training very large models on a relatively modest amount of data and stopping significantly before convergence.},
	urldate = {2026-01-12},
	publisher = {arXiv},
	author = {Kaplan, Jared and McCandlish, Sam and Henighan, Tom and Brown, Tom B. and Chess, Benjamin and Child, Rewon and Gray, Scott and Radford, Alec and Wu, Jeffrey and Amodei, Dario},
	month = jan,
	year = {2020},
	note = {arXiv:2001.08361 [cs]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
}
