@article{Acemoglu-RegulatingTransformativeTechnologies,
  title = {Regulating {{Transformative Technologies}}},
  author = {Acemoglu, Daron and Lensman, Todd},
  abstract = {Transformative technologies like generative artificial intelligence promise to accelerate productivity growth across many sectors, but they also present new risks from potential misuse. We develop a multi-sector technology adoption model to study the optimal regulation of transformative technologies when society can learn about these risks over time. Socially optimal adoption is gradual and convex. If social damages are proportional to the productivity gains from the new technology, a higher growth rate leads to slower optimal adoption. Equilibrium adoption is inefficient when firms do not internalize all social damages, and sector-independent regulation is helpful but generally not sufficient to restore optimality.},
  langid = {english},
  file = {/Users/boreas/Zotero/storage/NA5MYQGB/Acemoglu and Lensman - Regulating Transformative Technologies.pdf}
}

@inproceedings{Arjovsky2017-WassersteinGenerativeAdversarialNetworks,
  title = {Wasserstein {{Generative Adversarial Networks}}},
  booktitle = {Proceedings of the 34th {{International Conference}} on {{Machine Learning}}},
  author = {Arjovsky, Martin and Chintala, Soumith and Bottou, L{\'e}on},
  year = {2017},
  month = jul,
  pages = {214--223},
  publisher = {PMLR},
  issn = {2640-3498},
  urldate = {2024-12-13},
  abstract = {We introduce a new algorithm named WGAN, an alternative to traditional GAN training. In this new model, we show that we can improve the stability of learning, get rid of problems like mode collapse, and provide meaningful learning curves useful for debugging and hyperparameter searches. Furthermore, we show that the corresponding optimization problem is sound, and provide extensive theoretical work highlighting the deep connections to different distances between distributions.},
  langid = {english},
  file = {/Users/boreas/Zotero/storage/5HLFJ4SK/Arjovsky et al. - 2017 - Wasserstein Generative Adversarial Networks.pdf;/Users/boreas/Zotero/storage/YTUSBYB7/Arjovsky et al. - 2017 - Wasserstein Generative Adversarial Networks.pdf}
}

@article{Authors-MaxMinRLHFEquitableAlignmentLargeLanguage,
  title = {{{MaxMin-RLHF}}: {{Towards Equitable Alignment}} of {{Large Language Models}} with {{Diverse Human Preferences}}},
  author = {Authors, Anonymous},
  abstract = {Reinforcement Learning from Human Feedback (RLHF) aligns language models to human preferences by employing a singular reward model derived from preference data. However, the single reward model overlooks the rich diversity of human preferences inherent in data collected from multiple users. In this work, we first derive an impossibility result of alignment with single reward RLHF, thereby highlighting its insufficiency in representing diverse human preferences. Next, we propose to learn a mixture of reward models via an expectation-maximization algorithm and solve a MaxMin alignment objective inspired by the Egalitarian principle in social choice theory to better honor diverse human preferences. We present comprehensive experimental results on small-scale (GPT-2) and large-scale language (with Tulu2-7B)) and show the efficacy of the proposed approach in the presence of diversity among human preferences. We remark that our findings in this work are not only limited to language models but also extend to reinforcement learning in general.},
  langid = {english},
  file = {/Users/boreas/Zotero/storage/R9EMAPAL/Authors - MaxMin-RLHF Towards Equitable Alignment of Large Language Models with Diverse Human Preferences.pdf}
}

@misc{Cahyawijaya2024-HighDimensionHumanValueRepresentationLarge,
  title = {High-{{Dimension Human Value Representation}} in {{Large Language Models}}},
  author = {Cahyawijaya, Samuel and Chen, Delong and Bang, Yejin and Khalatbari, Leila and Wilie, Bryan and Ji, Ziwei and Ishii, Etsuko and Fung, Pascale},
  year = {2024},
  month = oct,
  number = {arXiv:2404.07900},
  eprint = {2404.07900},
  publisher = {arXiv},
  doi = {10.48550/arXiv.2404.07900},
  urldate = {2024-10-09},
  abstract = {The widespread application of Large Language Models (LLMs) across various tasks and fields has necessitated the alignment of these models with human values and preferences. Given various approaches of human value alignment, ranging from Reinforcement Learning with Human Feedback (RLHF), to constitutional learning, etc. there is an urgent need to understand the scope and nature of human values injected into these models before their release. There is also a need for model alignment without a costly large scale human annotation effort. We propose UniVaR, a high-dimensional representation of human value distributions in LLMs, orthogonal to model architecture and training data. Trained from the value-relevant output of eight multilingual LLMs and tested on the output from four multilingual LLMs, namely LlaMA2, ChatGPT, JAIS and Yi, we show that UniVaR is a powerful tool to compare the distribution of human values embedded in different LLMs with different langauge sources. Through UniVaR, we explore how different LLMs prioritize various values in different languages and cultures, shedding light on the complex interplay between human values and language modeling.},
  archiveprefix = {arXiv},
  keywords = {Computer Science - Artificial Intelligence,Computer Science - Computation and Language},
  file = {/Users/boreas/Zotero/storage/M43NPVTN/CahyawijayaChenBang2024-High-Dimension Human Value Representation in Large Language Models.pdf;/Users/boreas/Zotero/storage/SRX9FHAY/2404.html}
}

@inproceedings{Chakraborty2024-MaxMinRLHFAlignmentDiverseHumanPreferences,
  title = {{{MaxMin-RLHF}}: {{Alignment}} with {{Diverse Human Preferences}}},
  shorttitle = {{{MaxMin-RLHF}}},
  booktitle = {Forty-First {{International Conference}} on {{Machine Learning}}},
  author = {Chakraborty, Souradip and Qiu, Jiahao and Yuan, Hui and Koppel, Alec and Manocha, Dinesh and Huang, Furong and Bedi, Amrit and Wang, Mengdi},
  year = {2024},
  month = jun,
  urldate = {2024-12-13},
  abstract = {Reinforcement Learning from Human Feedback (RLHF) aligns language models to human preferences by employing a singular reward model derived from preference data. However, the single reward model overlooks the rich diversity of human preferences inherent in data collected from multiple users. In this work, we first derive an impossibility result of alignment with single reward RLHF, thereby highlighting its insufficiency in representing diverse human preferences. Next, we propose to learn a mixture of reward models via an expectation-maximization algorithm and solve a MaxMin alignment objective inspired by the Egalitarian principle in social choice theory to better honor diverse human preferences. We present comprehensive experimental results on small-scale (GPT-2) and large-scale language (with Tulu2-7B)) and show the efficacy of the proposed approach in the presence of diversity among human preferences. We remark that our findings in this work are not only limited to language models but also extend to reinforcement learning in general.},
  langid = {english},
  file = {/Users/boreas/Zotero/storage/LM8GHZBU/Chakraborty et al. - 2024 - MaxMin-RLHF Alignment with Diverse Human Preferences.pdf}
}

@article{Chen-ModelingPluralityHumanPreferencesIdeal,
  title = {Modeling the {{Plurality}} of {{Human Preferences}} via {{Ideal Points}}},
  author = {Chen, Daiwei and Chen, Yi and Rege, Aniket and Vinayak, Ramya Korlakai},
  abstract = {Large foundation models require extensive alignment to human preferences (Ouyang et al., 2022) before deployment. Existing methods typically collect large amounts of pairwise comparisons from humans (``Do you prefer output A or B?'') and utilize the Bradley-Terry-Luce (BTL) model (Bradley \& Terry, 1952) and generally suffer from assuming a universal preference shared by all humans, which lacks the flexibility to adapt to a plurality of opinions and preferences (Durmus et al., 2024). In this work, we propose PAL, a framework to model human preference complementary to existing pretraining strategies, which incorporates plurality from the ground up. We propose using the ideal point model (Coombs, 1950) as a lens to view alignment using preference comparisons. Together with our novel reformulation and using mixture modeling, our framework captures the plurality of population preferences while simultaneously learning a common preference latent space across different preferences, which can few-shot generalize to new, unseen users. With simple multi-layer perceptron, PAL achieves competitive reward model accuracy on Summary (Stiennon et al., 2020) (language), Picka-Pic (Kirstain et al., 2024) (image generation), and Persona (Perez et al., 2022) (semi-synthetic) induced heterogeneous datasets, matching stateof-the-art performance with greater efficiency. Finally, our experiments also highlight the shortcomings of current preference datasets created using rigid rubrics that wash away heterogeneity, and we call for more nuanced data collection approaches.},
  langid = {english},
  file = {/Users/boreas/Zotero/storage/3Y4EHCV8/Chen et al. - Modeling the Plurality of Human Preferences via Ideal Points.pdf}
}

@misc{Chen2024-PALPluralisticAlignmentFrameworkLearning,
  title = {{{PAL}}: {{Pluralistic Alignment Framework}} for {{Learning}} from {{Heterogeneous Preferences}}},
  shorttitle = {{{PAL}}},
  author = {Chen, Daiwei and Chen, Yi and Rege, Aniket and Vinayak, Ramya Korlakai},
  year = {2024},
  month = jun,
  number = {arXiv:2406.08469},
  eprint = {2406.08469},
  primaryclass = {cs},
  publisher = {arXiv},
  urldate = {2024-10-13},
  abstract = {Large foundation models pretrained on raw web-scale data are not readily deployable without additional step of extensive alignment to human preferences [1]. Such alignment is typically done by collecting large amounts of pairwise comparisons from humans (``Do you prefer output A or B?'') and learning a reward model or a policy with the Bradley-Terry-Luce (BTL) model [2] as a proxy for a human's underlying implicit preferences. These methods generally suffer from assuming a universal preference shared by all humans, which lacks the flexibility of adapting to plurality of opinions and preferences [3]. In this work, we propose PAL, a framework to model human preference complementary to existing pretraining strategies, which incorporates plurality from the ground up. We propose using the ideal point model [4] as a lens to view alignment using preference comparisons. Together with our novel reformulation and using mixture modeling, our framework captures the plurality of population preferences while simultaneously learning a common preference latent space across different preferences, which can few-shot generalize to new, unseen users. Our approach enables us to use the penultimate-layer representation of large foundation models and simple MLP layers to learn reward functions that are on-par with the existing large state-of-theart reward models, thereby enhancing efficiency of reward modeling significantly. We show that PAL achieves competitive reward model accuracy compared to strong baselines on 1) Language models with Summary dataset [5] ; 2) Image Generative models with Pick-a-Pic dataset [6] ; 3) A new semisynthetic heterogeneous dataset generated using Anthropic Personas [7]. Finally, our experiments also highlight the shortcoming of current preference datasets that are created using rigid rubrics which wash away heterogeneity, and call for more nuanced data collection approaches.},
  archiveprefix = {arXiv},
  langid = {english},
  keywords = {Computer Science - Machine Learning},
  file = {/Users/boreas/Zotero/storage/7GPKYYPI/Chen et al. - 2024 - PAL Pluralistic Alignment Framework for Learning from Heterogeneous Preferences.pdf}
}

@misc{Cui2024-UltraFeedbackBoostingLanguageModelsScaled,
  title = {{{UltraFeedback}}: {{Boosting Language Models}} with {{Scaled AI Feedback}}},
  shorttitle = {{{UltraFeedback}}},
  author = {Cui, Ganqu and Yuan, Lifan and Ding, Ning and Yao, Guanming and He, Bingxiang and Zhu, Wei and Ni, Yuan and Xie, Guotong and Xie, Ruobing and Lin, Yankai and Liu, Zhiyuan and Sun, Maosong},
  year = {2024},
  month = jul,
  number = {arXiv:2310.01377},
  eprint = {2310.01377},
  primaryclass = {cs},
  publisher = {arXiv},
  urldate = {2024-10-13},
  abstract = {Learning from human feedback has become a pivot technique in aligning large language models (LLMs) with human preferences. However, acquiring vast and premium human feedback is bottlenecked by time, labor, and human capability, resulting in small sizes or limited topics of current datasets. This further hinders feedback learning as well as alignment research within the open-source community. To address this issue, we explore how to go beyond human feedback and collect high-quality AI feedback automatically for a scalable alternative. Specifically, we identify scale and diversity as the key factors for feedback data to take effect. Accordingly, we first broaden instructions and responses in both amount and breadth to encompass a wider range of user-assistant interactions. Then, we meticulously apply a series of techniques to mitigate annotation biases for more reliable AI feedback. We finally present ULTRAFEEDBACK, a large-scale, high-quality, and diversified AI feedback dataset, which contains over 1 million GPT-4 feedback for 250k user-assistant conversations from various aspects. Built upon ULTRAFEEDBACK, we align a LLaMA-based model by best-of-n sampling and reinforcement learning, demonstrating its exceptional performance on chat benchmarks. Our work validates the effectiveness of scaled AI feedback data in constructing strong open-source chat language models, serving as a solid foundation for future feedback learning research.},
  archiveprefix = {arXiv},
  langid = {english},
  keywords = {Computer Science - Artificial Intelligence,Computer Science - Computation and Language,Computer Science - Machine Learning},
  file = {/Users/boreas/Zotero/storage/3DVIER9M/Cui et al. - 2024 - UltraFeedback Boosting Language Models with Scaled AI Feedback.pdf}
}

@misc{DOosterlinck2024-AnchoredPreferenceOptimizationContrastiveRevisions,
  title = {Anchored {{Preference Optimization}} and {{Contrastive Revisions}}: {{Addressing Underspecification}} in {{Alignment}}},
  shorttitle = {Anchored {{Preference Optimization}} and {{Contrastive Revisions}}},
  author = {D'Oosterlinck, Karel and Xu, Winnie and Develder, Chris and Demeester, Thomas and Singh, Amanpreet and Potts, Christopher and Kiela, Douwe and Mehri, Shikib},
  year = {2024},
  month = sep,
  number = {arXiv:2408.06266},
  eprint = {2408.06266},
  primaryclass = {cs},
  publisher = {arXiv},
  urldate = {2024-10-13},
  abstract = {Large Language Models (LLMs) are often aligned using contrastive alignment objectives and preference pair datasets. The interaction between model, paired data, and objective makes alignment a complicated procedure, sometimes producing subpar results. We study this and find that (i) preference data gives a better learning signal when the underlying responses are contrastive, and (ii) alignment objectives lead to better performance when they specify more control over the model during training. Based on these insights, we introduce Contrastive Learning from AI Revisions (CLAIR), a data-creation method which leads to more contrastive preference pairs, and Anchored Preference Optimization (APO), a controllable and more stable alignment objective. We align Llama-3-8B-Instruct using various comparable datasets and alignment objectives and measure MixEval-Hard scores, which correlate highly with human judgments. The CLAIR preferences lead to the strongest performance out of all datasets, and APO consistently outperforms less controllable objectives. Our best model, trained on 32K CLAIR preferences with APO, improves Llama-3-8B-Instruct by 7.65\%, closing the gap with GPT4-turbo by 45\%.},
  archiveprefix = {arXiv},
  langid = {english},
  keywords = {Computer Science - Artificial Intelligence,Computer Science - Computation and Language,Computer Science - Machine Learning},
  file = {/Users/boreas/Zotero/storage/UW9KWWBK/D'Oosterlinck et al. - 2024 - Anchored Preference Optimization and Contrastive Revisions Addressing Underspecification in Alignme.pdf}
}

@misc{Dwaracherla2024-EfficientExplorationLLMs,
  title = {Efficient {{Exploration}} for {{LLMs}}},
  author = {Dwaracherla, Vikranth and Asghari, Seyed Mohammad and Hao, Botao and Roy, Benjamin Van},
  year = {2024},
  month = jun,
  number = {arXiv:2402.00396},
  eprint = {2402.00396},
  doi = {10.48550/arXiv.2402.00396},
  urldate = {2024-10-24},
  abstract = {We present evidence of substantial benefit from efficient exploration in gathering human feedback to improve large language models. In our experiments, an agent sequentially generates queries while fitting a reward model to the feedback received. Our best-performing agent generates queries using double Thompson sampling, with uncertainty represented by an epistemic neural network. Our results demonstrate that efficient exploration enables high levels of performance with far fewer queries. Further, both uncertainty estimation and the choice of exploration scheme play critical roles.},
  archiveprefix = {arXiv},
  keywords = {Computer Science - Artificial Intelligence,Computer Science - Computation and Language,Computer Science - Machine Learning,Statistics - Machine Learning,Statistics - Methodology},
  file = {/Users/boreas/Zotero/storage/8Z8BHYGW/Dwaracherla et al. - 2024 - Efficient Exploration for LLMs.pdf}
}

@article{Eisenstein2024-RewardModelEnsemblesMitigatenot,
  title = {Reward {{Model Ensembles Mitigate}} but Do Not {{Eliminate Re-}} Ward {{Hacking}}},
  author = {Eisenstein, Jacob and Nagpal, Chirag and Agarwal, Alekh and Beirami, Ahmad and D'Amour, Alex and Dvijotham, {\relax DJ} and Fisch, Adam and Heller, Katherine and Pfohl, Stephen and Ramachandran, Deepak and Shaw, Peter and Berant, Jonathan},
  year = {2024},
  abstract = {Reward models play a key role in aligning language model applications towards human preferences. However, this setup creates an incentive for the language model to exploit errors in the reward model to achieve high estimated reward, a phenomenon often termed reward hacking. A natural mitigation is to train an ensemble of reward models, aggregating over model outputs to obtain a more robust reward estimate. We explore the application of reward ensembles to alignment at both training time (through reinforcement learning) and inference time (through reranking). First, we show that reward models are underspecified: reward models that perform similarly in-distribution can yield very different rewards when used in alignment, due to distribution shift. Second, underspecification results in overoptimization, where alignment to one reward model does not improve reward as measured by another reward model trained on the same data. Third, overoptimization is mitigated by the use of reward ensembles, and ensembles that vary by their pretraining seeds lead to better generalization than ensembles that differ only by their fine-tuning seeds, with both outperforming individual reward models.1 However, even pretrain reward ensembles do not eliminate reward hacking: we show several qualitative reward hacking phenomena that are not mitigated by ensembling because all reward models in the ensemble exhibit similar error patterns.},
  langid = {english},
  file = {/Users/boreas/Zotero/storage/9ZZTBNV4/Eisenstein et al. - 2024 - Reward Model Ensembles Mitigate but do not Eliminate Re- ward Hacking.pdf}
}

@misc{Ethayarajh2024-KTOModelAlignmentProspectTheoretic,
  title = {{{KTO}}: {{Model Alignment}} as {{Prospect Theoretic Optimization}}},
  shorttitle = {{{KTO}}},
  author = {Ethayarajh, Kawin and Xu, Winnie and Muennighoff, Niklas and Jurafsky, Dan and Kiela, Douwe},
  year = {2024},
  month = nov,
  number = {arXiv:2402.01306},
  eprint = {2402.01306},
  primaryclass = {cs},
  publisher = {arXiv},
  doi = {10.48550/arXiv.2402.01306},
  urldate = {2024-12-13},
  abstract = {Kahneman \& Tversky's \${\textbackslash}textit\{prospect theory\}\$ tells us that humans perceive random variables in a biased but well-defined manner (1992); for example, humans are famously loss-averse. We show that objectives for aligning LLMs with human feedback implicitly incorporate many of these biases -- the success of these objectives (e.g., DPO) over cross-entropy minimization can partly be ascribed to them belonging to a family of loss functions that we call \${\textbackslash}textit\{human-aware losses\}\$ (HALOs). However, the utility functions these methods attribute to humans still differ from those in the prospect theory literature. Using a Kahneman-Tversky model of human utility, we propose a HALO that directly maximizes the utility of generations instead of maximizing the log-likelihood of preferences, as current methods do. We call this approach KTO, and it matches or exceeds the performance of preference-based methods at scales from 1B to 30B, despite only learning from a binary signal of whether an output is desirable. More broadly, our work suggests that there is no one HALO that is universally superior; the best loss depends on the inductive biases most appropriate for a given setting, an oft-overlooked consideration.},
  archiveprefix = {arXiv},
  keywords = {Computer Science - Artificial Intelligence,Computer Science - Machine Learning},
  file = {/Users/boreas/Zotero/storage/463YMT77/Ethayarajh et al. - 2024 - KTO Model Alignment as Prospect Theoretic Optimization.pdf;/Users/boreas/Zotero/storage/C8YBK5MN/Ethayarajh et al. - 2024 - KTO Model Alignment as Prospect Theoretic Optimization.pdf}
}

@misc{Gao2024-UnifiedViewPreferenceLearningLarge,
  title = {Towards a {{Unified View}} of {{Preference Learning}} for {{Large Language Models}}: {{A Survey}}},
  shorttitle = {Towards a {{Unified View}} of {{Preference Learning}} for {{Large Language Models}}},
  author = {Gao, Bofei and Song, Feifan and Miao, Yibo and Cai, Zefan and Yang, Zhe and Chen, Liang and Hu, Helan and Xu, Runxin and Dong, Qingxiu and Zheng, Ce and Xiao, Wen and Zhang, Ge and Zan, Daoguang and Lu, Keming and Yu, Bowen and Liu, Dayiheng and Cui, Zeyu and Yang, Jian and Sha, Lei and Wang, Houfeng and Sui, Zhifang and Wang, Peiyi and Liu, Tianyu and Chang, Baobao},
  year = {2024},
  month = sep,
  number = {arXiv:2409.02795},
  eprint = {2409.02795},
  primaryclass = {cs},
  publisher = {arXiv},
  urldate = {2024-10-13},
  abstract = {Large Language Models (LLMs) exhibit remarkably powerful capabilities. One of the crucial factors to achieve success is aligning the LLM's output with human preferences. This alignment process often requires only a small amount of data to efficiently enhance the LLM's performance. While effective, research in this area spans multiple domains, and the methods involved are relatively complex to understand. The relationships between different methods have been under-explored, limiting the development of the preference alignment. In light of this, we break down the existing popular alignment strategies into different components and provide a unified framework to study the current alignment strategies, thereby establishing connections among them. In this survey, we decompose all the strategies in preference learning into four components: model, data, feedback, and algorithm. This unified view offers an in-depth understanding of existing alignment algorithms and also opens up possibilities to synergize the strengths of different strategies. Furthermore, we present detailed working examples of prevalent existing algorithms to facilitate a comprehensive understanding for the readers. Finally, based on our unified perspective, we explore the challenges and future research directions for aligning large language models with human preferences.},
  archiveprefix = {arXiv},
  langid = {english},
  keywords = {Computer Science - Computation and Language},
  file = {/Users/boreas/Zotero/storage/NYBQPCKS/Gao et al. - 2024 - Towards a Unified View of Preference Learning for Large Language Models A Survey.pdf}
}

@misc{Ji2023-BeaverTailsImprovedSafetyAlignmentLLM,
  title = {{{BeaverTails}}: {{Towards Improved Safety Alignment}} of {{LLM}} via a {{Human-Preference Dataset}}},
  shorttitle = {{{BeaverTails}}},
  author = {Ji, Jiaming and Liu, Mickel and Dai, Juntao and Pan, Xuehai and Zhang, Chi and Bian, Ce and Zhang, Chi and Sun, Ruiyang and Wang, Yizhou and Yang, Yaodong},
  year = {2023},
  month = nov,
  number = {arXiv:2307.04657},
  eprint = {2307.04657},
  primaryclass = {cs},
  publisher = {arXiv},
  urldate = {2024-10-13},
  abstract = {In this paper, we introduce the BEAVERTAILS dataset, aimed at fostering research on safety alignment in large language models (LLMs). This dataset uniquely separates annotations of helpfulness and harmlessness for question-answering pairs, thus offering distinct perspectives on these crucial attributes. In total, we have gathered safety meta-labels for 333,963 question-answer (QA) pairs and 361,903 pairs of expert comparison data for both the helpfulness and harmlessness metrics. We further showcase applications of BeaverTails in content moderation and reinforcement learning with human feedback (RLHF), emphasizing its potential for practical safety measures in LLMs. We believe this dataset provides vital resources for the community, contributing towards the safe development and deployment of LLMs. Our project page is available at the following URL: https://sites. google.com/view/pku-beavertails. Warning: this paper contains example data that may be offensive or harmful.},
  archiveprefix = {arXiv},
  langid = {english},
  keywords = {Computer Science - Computation and Language},
  file = {/Users/boreas/Zotero/storage/5AESEJ77/Ji et al. - 2023 - BeaverTails Towards Improved Safety Alignment of LLM via a Human-Preference Dataset.pdf}
}

@misc{Lambert2024-RewardBenchEvaluatingRewardModelsLanguage,
  title = {{{RewardBench}}: {{Evaluating Reward Models}} for {{Language Modeling}}},
  shorttitle = {{{RewardBench}}},
  author = {Lambert, Nathan and Pyatkin, Valentina and Morrison, Jacob and Miranda, L. J. and Lin, Bill Yuchen and Chandu, Khyathi and Dziri, Nouha and Kumar, Sachin and Zick, Tom and Choi, Yejin and Smith, Noah A. and Hajishirzi, Hannaneh},
  year = {2024},
  month = jun,
  number = {arXiv:2403.13787},
  eprint = {2403.13787},
  primaryclass = {cs},
  publisher = {arXiv},
  urldate = {2024-10-13},
  abstract = {Reward models (RMs) are at the crux of successfully using RLHF to align pretrained models to human preferences, yet there has been relatively little study that focuses on evaluation of those models. Evaluating reward models presents an opportunity to understand the opaque technologies used for alignment of language models and which values are embedded in them. Resources for reward model training and understanding are sparse in the nascent open-source community around them. To enhance scientific understanding of reward models, we present REWARDBENCH, a benchmark dataset and code-base for evaluation. The REWARDBENCH dataset is a collection of prompt-chosen-rejected trios spanning chat, reasoning, and safety, to benchmark how reward models perform on challenging, structured and out-of-distribution queries. We create specific comparison datasets for RMs that have subtle, but verifiable reasons (e.g. bugs, incorrect facts) why one answer should be preferred to another. On the REWARDBENCH leaderboard, we evaluate reward models trained with a variety of methods, such as the direct MLE training of classifiers and the implicit reward modeling of Direct Preference Optimization (DPO). We present many findings on propensity for refusals, reasoning limitations, and instruction following shortcomings of various reward models towards a better understanding of the RLHF process.},
  archiveprefix = {arXiv},
  langid = {english},
  keywords = {Computer Science - Machine Learning},
  file = {/Users/boreas/Zotero/storage/NQTKAAM9/Lambert et al. - 2024 - RewardBench Evaluating Reward Models for Language Modeling.pdf}
}

@misc{Liu2024-HowLargeLanguageModelsNavigate,
  title = {How Do {{Large Language Models Navigate Conflicts}} between {{Honesty}} and {{Helpfulness}}?},
  author = {Liu, Ryan and Sumers, Theodore R. and Dasgupta, Ishita and Griffiths, Thomas L.},
  year = {2024},
  month = feb,
  number = {arXiv:2402.07282},
  eprint = {2402.07282},
  publisher = {arXiv},
  doi = {10.48550/arXiv.2402.07282},
  urldate = {2024-10-10},
  abstract = {In day-to-day communication, people often approximate the truth - for example, rounding the time or omitting details - in order to be maximally helpful to the listener. How do large language models (LLMs) handle such nuanced trade-offs? To address this question, we use psychological models and experiments designed to characterize human behavior to analyze LLMs. We test a range of LLMs and explore how optimization for human preferences or inference-time reasoning affects these trade-offs. We find that reinforcement learning from human feedback improves both honesty and helpfulness, while chain-of-thought prompting skews LLMs towards helpfulness over honesty. Finally, GPT-4 Turbo demonstrates human-like response patterns including sensitivity to the conversational framing and listener's decision context. Our findings reveal the conversational values internalized by LLMs and suggest that even these abstract values can, to a degree, be steered by zero-shot prompting.},
  archiveprefix = {arXiv},
  keywords = {Computer Science - Artificial Intelligence,Computer Science - Computation and Language,Computer Science - Machine Learning},
  file = {/Users/boreas/Zotero/storage/RUTZ27PW/LiuSumersDasgupta2024-How do Large Language Models Navigate Conflicts between Honesty and Helpfulness.pdf;/Users/boreas/Zotero/storage/Z4XLWJWY/2402.html}
}

@misc{Liu2024-KANKolmogorovArnoldNetworks,
  title = {{{KAN}}: {{Kolmogorov-Arnold Networks}}},
  shorttitle = {{{KAN}}},
  author = {Liu, Ziming and Wang, Yixuan and Vaidya, Sachin and Ruehle, Fabian and Halverson, James and Solja{\v c}i{\'c}, Marin and Hou, Thomas Y. and Tegmark, Max},
  year = {2024},
  month = jun,
  number = {arXiv:2404.19756},
  eprint = {2404.19756},
  primaryclass = {cond-mat, stat},
  publisher = {arXiv},
  urldate = {2024-06-18},
  abstract = {Inspired by the Kolmogorov-Arnold representation theorem, we propose KolmogorovArnold Networks (KANs) as promising alternatives to Multi-Layer Perceptrons (MLPs). While MLPs have fixed activation functions on nodes (``neurons''), KANs have learnable activation functions on edges (``weights''). KANs have no linear weights at all -- every weight parameter is replaced by a univariate function parametrized as a spline. We show that this seemingly simple change makes KANs outperform MLPs in terms of accuracy and interpretability, on small-scale AI + Science tasks. For accuracy, smaller KANs can achieve comparable or better accuracy than larger MLPs in function fitting tasks. Theoretically and empirically, KANs possess faster neural scaling laws than MLPs. For interpretability, KANs can be intuitively visualized and can easily interact with human users. Through two examples in mathematics and physics, KANs are shown to be useful ``collaborators'' helping scientists (re)discover mathematical and physical laws. In summary, KANs are promising alternatives for MLPs, opening opportunities for further improving today's deep learning models which rely heavily on MLPs.},
  archiveprefix = {arXiv},
  langid = {english},
  keywords = {Computer Science - Artificial Intelligence,Computer Science - Machine Learning,Condensed Matter - Disordered Systems and Neural Networks,Statistics - Machine Learning},
  file = {/Users/boreas/Zotero/storage/KD778VMB/LiuWangVaidya2024-KAN - Kolmogorov-Arnold Networks.pdf;/Users/boreas/Zotero/storage/MHCEHBA9/LiuWangVaidya2024-KAN - Kolmogorov-Arnold Networks.pdf}
}

@article{Moon2019-Visualizingstructuretransitionshighdimensionalbiologicala,
  title = {Visualizing Structure and Transitions in High-Dimensional Biological Data},
  author = {Moon, Kevin R. and {van Dijk}, David and Wang, Zheng and Gigante, Scott and Burkhardt, Daniel B. and Chen, William S. and Yim, Kristina and van den Elzen, Antonia and Hirn, Matthew J. and Coifman, Ronald R. and Ivanova, Natalia B. and Wolf, Guy and Krishnaswamy, Smita},
  year = {2019},
  month = dec,
  journal = {Nature Biotechnology},
  volume = {37},
  number = {12},
  pages = {1482--1492},
  publisher = {Nature Publishing Group},
  issn = {1546-1696},
  doi = {10.1038/s41587-019-0336-3},
  urldate = {2023-04-24},
  abstract = {The high-dimensional data created by high-throughput technologies require visualization tools that reveal data structure and patterns in an intuitive form. We present PHATE, a visualization method that captures both local and global nonlinear structure using an information-geometric distance between data points. We compare PHATE to other tools on a variety of artificial and biological datasets, and find that it consistently preserves a range of patterns in data, including continual progressions, branches and clusters, better than other tools. We define a manifold preservation metric, which we call denoised embedding manifold preservation (DEMaP), and show that PHATE produces lower-dimensional embeddings that are quantitatively better denoised as compared to existing visualization methods. An analysis of a newly generated single-cell RNA sequencing dataset on human germ-layer differentiation demonstrates how PHATE reveals unique biological insight into the main developmental branches, including identification of three previously undescribed subpopulations. We also show that PHATE is applicable to a wide variety of data types, including mass cytometry, single-cell RNA sequencing, Hi-C and gut microbiome data.},
  copyright = {2019 The Author(s), under exclusive licence to Springer Nature America, Inc.},
  langid = {english},
  keywords = {Data mining,Machine learning},
  file = {/Users/boreas/Zotero/storage/F3KQK88H/Moonvan DijkWang2019-Visualizing structure and transitions in high-dimensional biological data2.pdf;/Users/boreas/Zotero/storage/PGQX3H5C/Moonvan DijkWang2019-Visualizing structure and transitions in high-dimensional biological data2.pdf;/Users/boreas/Zotero/storage/Q6LY7TWD/Moonvan DijkWang2019-Visualizing structure and transitions in high-dimensional biological data2.pdf}
}

@misc{Osband2023-EpistemicNeuralNetworks,
  title = {Epistemic {{Neural Networks}}},
  author = {Osband, Ian and Wen, Zheng and Asghari, Seyed Mohammad and Dwaracherla, Vikranth and Ibrahimi, Morteza and Lu, Xiuyuan and Roy, Benjamin Van},
  year = {2023},
  month = may,
  number = {arXiv:2107.08924},
  eprint = {2107.08924},
  publisher = {arXiv},
  doi = {10.48550/arXiv.2107.08924},
  urldate = {2024-11-01},
  abstract = {Intelligence relies on an agent's knowledge of what it does not know. This capability can be assessed based on the quality of joint predictions of labels across multiple inputs. In principle, ensemble-based approaches produce effective joint predictions, but the computational costs of training large ensembles can become prohibitive. We introduce the epinet: an architecture that can supplement any conventional neural network, including large pretrained models, and can be trained with modest incremental computation to estimate uncertainty. With an epinet, conventional neural networks outperform very large ensembles, consisting of hundreds or more particles, with orders of magnitude less computation. The epinet does not fit the traditional framework of Bayesian neural networks. To accommodate development of approaches beyond BNNs, such as the epinet, we introduce the epistemic neural network (ENN) as an interface for models that produce joint predictions.},
  archiveprefix = {arXiv},
  keywords = {Computer Science - Artificial Intelligence,Computer Science - Machine Learning,Statistics - Machine Learning},
  file = {/Users/boreas/Zotero/storage/XB37C8X5/Osband et al. - 2023 - Epistemic Neural Networks.pdf;/Users/boreas/Zotero/storage/J6FE6J4C/2107.html}
}

@misc{Ouyang2022-Traininglanguagemodelsfollowinstructions,
  title = {Training Language Models to Follow Instructions with Human Feedback},
  author = {Ouyang, Long and Wu, Jeff and Jiang, Xu and Almeida, Diogo and Wainwright, Carroll L. and Mishkin, Pamela and Zhang, Chong and Agarwal, Sandhini and Slama, Katarina and Ray, Alex and Schulman, John and Hilton, Jacob and Kelton, Fraser and Miller, Luke and Simens, Maddie and Askell, Amanda and Welinder, Peter and Christiano, Paul and Leike, Jan and Lowe, Ryan},
  year = {2022},
  month = mar,
  journal = {arXiv.org},
  urldate = {2024-09-29},
  abstract = {Making language models bigger does not inherently make them better at following a user's intent. For example, large language models can generate outputs that are untruthful, toxic, or simply not helpful to the user. In other words, these models are not aligned with their users. In this paper, we show an avenue for aligning language models with user intent on a wide range of tasks by fine-tuning with human feedback. Starting with a set of labeler-written prompts and prompts submitted through the OpenAI API, we collect a dataset of labeler demonstrations of the desired model behavior, which we use to fine-tune GPT-3 using supervised learning. We then collect a dataset of rankings of model outputs, which we use to further fine-tune this supervised model using reinforcement learning from human feedback. We call the resulting models InstructGPT. In human evaluations on our prompt distribution, outputs from the 1.3B parameter InstructGPT model are preferred to outputs from the 175B GPT-3, despite having 100x fewer parameters. Moreover, InstructGPT models show improvements in truthfulness and reductions in toxic output generation while having minimal performance regressions on public NLP datasets. Even though InstructGPT still makes simple mistakes, our results show that fine-tuning with human feedback is a promising direction for aligning language models with human intent.},
  howpublished = {https://arxiv.org/abs/2203.02155v1},
  langid = {english},
  file = {/Users/boreas/Zotero/storage/2YNZU8NZ/OuyangWuJiang2022-Training language models to follow instructions with human feedback2.pdf;/Users/boreas/Zotero/storage/4EWXHEA4/OuyangWuJiang2022-Training language models to follow instructions with human feedback2.pdf}
}

@misc{Poddar2024-PersonalizingReinforcementLearningHumanFeedback,
  title = {Personalizing {{Reinforcement Learning}} from {{Human Feedback}} with {{Variational Preference Learning}}},
  author = {Poddar, Sriyash and Wan, Yanming and Ivison, Hamish and Gupta, Abhishek and Jaques, Natasha},
  year = {2024},
  month = aug,
  number = {arXiv:2408.10075},
  eprint = {2408.10075},
  primaryclass = {cs},
  publisher = {arXiv},
  urldate = {2024-10-13},
  abstract = {Reinforcement Learning from Human Feedback (RLHF) is a powerful paradigm for aligning foundation models to human values and preferences. However, current RLHF techniques cannot account for the naturally occurring differences in individual human preferences across a diverse population. When these differences arise, traditional RLHF frameworks simply average over them, leading to inaccurate rewards and poor performance for individual subgroups. To address the need for pluralistic alignment, we develop a class of multimodal RLHF methods. Our proposed techniques are based on a latent variable formulation - inferring a novel user-specific latent and learning reward models and policies conditioned on this latent without additional user-specific data. While conceptually simple, we show that in practice, this reward modeling requires careful algorithmic considerations around model architecture and reward scaling. To empirically validate our proposed technique, we first show that it can provide a way to combat underspecification in simulated control problems, inferring and optimizing user-specific reward functions. Next, we conduct experiments on pluralistic language datasets representing diverse user preferences and demonstrate improved reward function accuracy. We additionally show the benefits of this probabilistic framework in terms of measuring uncertainty, and actively learning user preferences. This work enables learning from diverse populations of users with divergent preferences, an important challenge that naturally occurs in problems from robot learning to foundation model alignment.},
  archiveprefix = {arXiv},
  langid = {english},
  keywords = {Computer Science - Artificial Intelligence,Computer Science - Computation and Language,Computer Science - Machine Learning,Computer Science - Robotics},
  file = {/Users/boreas/Zotero/storage/G8LUZRM8/Poddar et al. - 2024 - Personalizing Reinforcement Learning from Human Feedback with Variational Preference Learning.pdf}
}

@misc{Rafailov2024-DirectPreferenceOptimizationYourLanguageb,
  title = {Direct {{Preference Optimization}}: {{Your Language Model}} Is {{Secretly}} a {{Reward Model}}},
  shorttitle = {Direct {{Preference Optimization}}},
  author = {Rafailov, Rafael and Sharma, Archit and Mitchell, Eric and Ermon, Stefano and Manning, Christopher D. and Finn, Chelsea},
  year = {2024},
  month = jul,
  number = {arXiv:2305.18290},
  eprint = {2305.18290},
  primaryclass = {cs},
  publisher = {arXiv},
  doi = {10.48550/arXiv.2305.18290},
  urldate = {2024-12-13},
  abstract = {While large-scale unsupervised language models (LMs) learn broad world knowledge and some reasoning skills, achieving precise control of their behavior is difficult due to the completely unsupervised nature of their training. Existing methods for gaining such steerability collect human labels of the relative quality of model generations and fine-tune the unsupervised LM to align with these preferences, often with reinforcement learning from human feedback (RLHF). However, RLHF is a complex and often unstable procedure, first fitting a reward model that reflects the human preferences, and then fine-tuning the large unsupervised LM using reinforcement learning to maximize this estimated reward without drifting too far from the original model. In this paper we introduce a new parameterization of the reward model in RLHF that enables extraction of the corresponding optimal policy in closed form, allowing us to solve the standard RLHF problem with only a simple classification loss. The resulting algorithm, which we call Direct Preference Optimization (DPO), is stable, performant, and computationally lightweight, eliminating the need for sampling from the LM during fine-tuning or performing significant hyperparameter tuning. Our experiments show that DPO can fine-tune LMs to align with human preferences as well as or better than existing methods. Notably, fine-tuning with DPO exceeds PPO-based RLHF in ability to control sentiment of generations, and matches or improves response quality in summarization and single-turn dialogue while being substantially simpler to implement and train.},
  archiveprefix = {arXiv},
  keywords = {Computer Science - Artificial Intelligence,Computer Science - Computation and Language,Computer Science - Machine Learning},
  file = {/Users/boreas/Zotero/storage/IV77QX7N/Rafailov et al. - 2024 - Direct Preference Optimization Your Language Model is Secretly a Reward Model.pdf;/Users/boreas/Zotero/storage/DPZRCZKW/2305.html}
}

@misc{Rame2023-RewardedsoupsParetooptimalalignmentinterpolating,
  title = {Rewarded Soups: Towards {{Pareto-optimal}} Alignment by Interpolating Weights Fine-Tuned on Diverse Rewards},
  shorttitle = {Rewarded Soups},
  author = {Ram{\'e}, Alexandre and Couairon, Guillaume and Shukor, Mustafa and Dancette, Corentin and Gaya, Jean-Baptiste and Soulier, Laure and Cord, Matthieu},
  year = {2023},
  month = oct,
  number = {arXiv:2306.04488},
  eprint = {2306.04488},
  primaryclass = {cs},
  publisher = {arXiv},
  urldate = {2024-10-13},
  abstract = {Foundation models are first pre-trained on vast unsupervised datasets and then fine-tuned on labeled data. Reinforcement learning, notably from human feedback (RLHF), can further align the network with the intended usage. Yet the imperfections in the proxy reward may hinder the training and lead to suboptimal results; the diversity of objectives in real-world tasks and human opinions exacerbate the issue. This paper proposes embracing the heterogeneity of diverse rewards by following a multi-policy strategy. Rather than focusing on a single a priori reward, we aim for Pareto-optimal generalization across the entire space of preferences. To this end, we propose rewarded soup, first specializing multiple networks independently (one for each proxy reward) and then interpolating their weights linearly. This succeeds empirically because we show that the weights remain linearly connected when fine-tuned on diverse rewards from a shared pre-trained initialization. We demonstrate the effectiveness of our approach for text-to-text (summarization, Q\&A, helpful assistant, review), text-image (image captioning, text-to-image generation, visual grounding, VQA), and control (locomotion) tasks. We hope to enhance the alignment of deep models, and how they interact with the world in all its diversity.},
  archiveprefix = {arXiv},
  langid = {english},
  keywords = {Computer Science - Artificial Intelligence,Computer Science - Computer Vision and Pattern Recognition,Computer Science - Machine Learning},
  file = {/Users/boreas/Zotero/storage/YZ7ZEKYQ/Ramé et al. - 2023 - Rewarded soups towards Pareto-optimal alignment by interpolating weights fine-tuned on diverse rewa.pdf}
}

@inproceedings{Rame2024-WARMBenefitsWeightAveragedReward,
  title = {{{WARM}}: {{On}} the {{Benefits}} of {{Weight Averaged Reward Models}}},
  shorttitle = {{{WARM}}},
  booktitle = {Forty-First {{International Conference}} on {{Machine Learning}}, {{ICML}} 2024, {{Vienna}}, {{Austria}}, {{July}} 21-27, 2024},
  author = {Ram{\'e}, Alexandre and Vieillard, Nino and Hussenot, L{\'e}onard and Dadashi, Robert and Cideron, Geoffrey and Bachem, Olivier and Ferret, Johan},
  year = {2024},
  publisher = {OpenReview.net},
  urldate = {2024-12-13},
  file = {/Users/boreas/Zotero/storage/FI3CSX8W/Ramé et al. - 2024 - WARM On the Benefits of Weight Averaged Reward Models.pdf}
}

@misc{Shaikh2024-ShowDontTellAligningLanguage,
  title = {Show, {{Don}}'t {{Tell}}: {{Aligning Language Models}} with {{Demonstrated Feedback}}},
  shorttitle = {Show, {{Don}}'t {{Tell}}},
  author = {Shaikh, Omar and Lam, Michelle and Hejna, Joey and Shao, Yijia and Bernstein, Michael and Yang, Diyi},
  year = {2024},
  month = jun,
  number = {arXiv:2406.00888},
  eprint = {2406.00888},
  primaryclass = {cs},
  publisher = {arXiv},
  urldate = {2024-10-13},
  abstract = {Language models are aligned to emulate the collective voice of many, resulting in outputs that align with no one in particular. Steering LLMs away from generic output is possible through supervised finetuning or RLHF, but requires prohibitively large datasets for new ad-hoc tasks. We argue that it is instead possible to align an LLM to a specific setting by leveraging a very small number (\${$<$}10\$) of demonstrations as feedback. Our method, Demonstration ITerated Task Optimization (DITTO), directly aligns language model outputs to a user's demonstrated behaviors. Derived using ideas from online imitation learning, DITTO cheaply generates online comparison data by treating users' demonstrations as preferred over output from the LLM and its intermediate checkpoints. We evaluate DITTO's ability to learn fine-grained style and task alignment across domains such as news articles, emails, and blog posts. Additionally, we conduct a user study soliciting a range of demonstrations from participants (\$N=16\$). Across our benchmarks and user study, we find that win-rates for DITTO outperform few-shot prompting, supervised fine-tuning, and other self-play methods by an average of 19\% points. By using demonstrations as feedback directly, DITTO offers a novel method for effective customization of LLMs.},
  archiveprefix = {arXiv},
  langid = {english},
  keywords = {Computer Science - Computation and Language,Computer Science - Human-Computer Interaction},
  file = {/Users/boreas/Zotero/storage/BQRERQ3W/Shaikh et al. - 2024 - Show, Don't Tell Aligning Language Models with Demonstrated Feedback.pdf}
}

@inproceedings{Tennant2024-MoralAlignmentLLMAgents,
  title = {Moral {{Alignment}} for {{LLM Agents}}},
  author = {Tennant, Elizaveta and Hailes, Stephen and Musolesi, Mirco},
  year = {2024},
  month = oct,
  urldate = {2024-10-10},
  abstract = {Decision-making agents based on pre-trained Large Language Models (LLMs) are increasingly being deployed across various domains of human activity. While their applications are currently rather specialized, several research efforts are under way to develop more generalist agents. As LLM-based systems become more agentic, their influence on human activity will grow and the transparency of this will decrease. Consequently, developing effective methods for aligning them to human values is vital. The prevailing practice in alignment often relies on human preference data (e.g., in RLHF or DPO), in which values are implicit and are essentially deduced from relative preferences over different model outputs. In this work, instead of relying on human feedback, we introduce the design of reward functions that explicitly encode core human values for Reinforcement Learning-based fine-tuning of foundation agent models. Specifically, we use intrinsic rewards for the moral alignment of LLM agents. We evaluate our approach using the traditional philosophical frameworks of Deontological Ethics and Utilitarianism, quantifying moral rewards for agents in terms of actions and consequences on the Iterated Prisoner's Dilemma (IPD) environment. We also show how moral fine-tuning can be deployed to enable an agent to unlearn a previously developed selfish strategy. Finally, we find that certain moral strategies learned on the IPD game generalize to several other matrix game environments. In summary, we demonstrate that fine-tuning with intrinsic rewards is a promising general solution for aligning LLM agents to human values, and it might represent a more transparent and cost-effective alternative to currently predominant alignment techniques.},
  file = {/Users/boreas/Zotero/storage/EV9QM4KL/TennantHailesMusolesi2024-Moral Alignment for LLM Agents.pdf}
}

@inproceedings{Wang2024-ArithmeticControlLLMsDiverseUser,
  title = {Arithmetic {{Control}} of {{LLMs}} for {{Diverse User Preferences}}: {{Directional Preference Alignment}} with {{Multi-Objective Rewards}}},
  shorttitle = {Arithmetic {{Control}} of {{LLMs}} for {{Diverse User Preferences}}},
  booktitle = {Proceedings of the 62nd {{Annual Meeting}} of the {{Association}} for {{Computational Linguistics}} ({{Volume}} 1: {{Long Papers}}), {{ACL}} 2024, {{Bangkok}}, {{Thailand}}, {{August}} 11-16, 2024},
  author = {Wang, Haoxiang and Lin, Yong and Xiong, Wei and Yang, Rui and Diao, Shizhe and Qiu, Shuang and Zhao, Han and Zhang, Tong},
  editor = {Ku, Lun-Wei and Martins, Andre and Srikumar, Vivek},
  year = {2024},
  pages = {8642--8655},
  publisher = {Association for Computational Linguistics},
  doi = {10.18653/V1/2024.ACL-LONG.468},
  urldate = {2024-12-13},
  file = {/Users/boreas/Zotero/storage/I7TJBBTZ/Wang et al. - 2024 - Arithmetic Control of LLMs for Diverse User Preferences Directional Preference Alignment with Multi.pdf;/Users/boreas/Zotero/storage/XRUT6JR6/WangLinXiong2024-Arithmetic Control of LLMs for Diverse User Preferences.pdf;/Users/boreas/Zotero/storage/EWBSZRFI/2402.html}
}

@misc{Wang2024-InterpretablePreferencesMultiObjectiveRewardModelinga,
  title = {Interpretable {{Preferences}} via {{Multi-Objective Reward Modeling}} and {{Mixture-of-Experts}}},
  author = {Wang, Haoxiang and Xiong, Wei and Xie, Tengyang and Zhao, Han and Zhang, Tong},
  year = {2024},
  month = jun,
  number = {arXiv:2406.12845},
  eprint = {2406.12845},
  publisher = {arXiv},
  doi = {10.48550/arXiv.2406.12845},
  urldate = {2024-10-10},
  abstract = {Reinforcement learning from human feedback (RLHF) has emerged as the primary method for aligning large language models (LLMs) with human preferences. The RLHF process typically starts by training a reward model (RM) using human preference data. Conventional RMs are trained on pairwise responses to the same user request, with relative ratings indicating which response humans prefer. The trained RM serves as a proxy for human preferences. However, due to the black-box nature of RMs, their outputs lack interpretability, as humans cannot intuitively understand why an RM thinks a response is good or not. As RMs act as human preference proxies, we believe they should be human-interpretable to ensure that their internal decision processes are consistent with human preferences and to prevent reward hacking in LLM alignment. To build RMs with interpretable preferences, we propose a two-stage approach: i) train an Absolute-Rating Multi-Objective Reward Model (ArmoRM) with multi-dimensional absolute-rating data, each dimension corresponding to a human-interpretable objective (e.g., honesty, verbosity, safety); ii) employ a Mixture-of-Experts (MoE) strategy with a gating network that automatically selects the most suitable reward objectives based on the context. We efficiently trained an ArmoRM with Llama-3 8B and a gating network consisting of a shallow MLP on top of the ArmoRM. Our trained model, ArmoRM-Llama3-8B, obtains state-of-the-art performance on RewardBench, a benchmark evaluating RMs for language modeling. Notably, the performance of our model surpasses the LLM-as-a-judge method with GPT-4 judges by a margin, and approaches the performance of the much larger Nemotron-4 340B reward model.},
  archiveprefix = {arXiv},
  keywords = {Computer Science - Computation and Language,Computer Science - Machine Learning},
  file = {/Users/boreas/Zotero/storage/EVKMHV4R/WangXiongXie2024-Interpretable Preferences via Multi-Objective Reward Modeling and.pdf;/Users/boreas/Zotero/storage/8KXREDHJ/2406.html}
}

@misc{Wang2024-InterpretablePreferencesMultiObjectiveRewardModelingb,
  title = {Interpretable {{Preferences}} via {{Multi-Objective Reward Modeling}} and {{Mixture-of-Experts}}},
  author = {Wang, Haoxiang and Xiong, Wei and Xie, Tengyang and Zhao, Han and Zhang, Tong},
  year = {2024},
  month = jun,
  number = {arXiv:2406.12845},
  eprint = {2406.12845},
  publisher = {arXiv},
  doi = {10.48550/arXiv.2406.12845},
  urldate = {2024-10-13},
  abstract = {Reinforcement learning from human feedback (RLHF) has emerged as the primary method for aligning large language models (LLMs) with human preferences. The RLHF process typically starts by training a reward model (RM) using human preference data. Conventional RMs are trained on pairwise responses to the same user request, with relative ratings indicating which response humans prefer. The trained RM serves as a proxy for human preferences. However, due to the black-box nature of RMs, their outputs lack interpretability, as humans cannot intuitively understand why an RM thinks a response is good or not. As RMs act as human preference proxies, we believe they should be human-interpretable to ensure that their internal decision processes are consistent with human preferences and to prevent reward hacking in LLM alignment. To build RMs with interpretable preferences, we propose a two-stage approach: i) train an Absolute-Rating Multi-Objective Reward Model (ArmoRM) with multi-dimensional absolute-rating data, each dimension corresponding to a human-interpretable objective (e.g., honesty, verbosity, safety); ii) employ a Mixture-of-Experts (MoE) strategy with a gating network that automatically selects the most suitable reward objectives based on the context. We efficiently trained an ArmoRM with Llama-3 8B and a gating network consisting of a shallow MLP on top of the ArmoRM. Our trained model, ArmoRM-Llama3-8B, obtains state-of-the-art performance on RewardBench, a benchmark evaluating RMs for language modeling. Notably, the performance of our model surpasses the LLM-as-a-judge method with GPT-4 judges by a margin, and approaches the performance of the much larger Nemotron-4 340B reward model.},
  archiveprefix = {arXiv},
  keywords = {Computer Science - Computation and Language,Computer Science - Machine Learning},
  file = {/Users/boreas/Zotero/storage/L7WQ7JUB/Wang et al. - 2024 - Interpretable Preferences via Multi-Objective Reward Modeling and Mixture-of-Experts.pdf;/Users/boreas/Zotero/storage/X6H89Z77/2406.html}
}

@misc{Wang2024-TransformingCombiningRewardsAligningLarge,
  title = {Transforming and {{Combining Rewards}} for {{Aligning Large Language Models}}},
  author = {Wang, Zihao and Nagpal, Chirag and Berant, Jonathan and Eisenstein, Jacob and D'Amour, Alex and Koyejo, Sanmi and Veitch, Victor},
  year = {2024},
  month = jul,
  number = {arXiv:2402.00742},
  eprint = {2402.00742},
  primaryclass = {cs},
  publisher = {arXiv},
  urldate = {2024-10-13},
  abstract = {A common approach for aligning language models to human preferences is to first learn a reward model from preference data, and then use this reward model to update the language model. We study two closely related problems that arise in this approach. First, any monotone transformation of the reward model preserves preference ranking; is there a choice that is ``better'' than others? Second, we often wish to align language models to multiple properties: how should we combine multiple reward models? Using a probabilistic interpretation of the alignment procedure, we identify a natural choice for transformation for (the common case of) rewards learned from Bradley-Terry preference models. The derived transformation is straightforward: we apply a log-sigmoid function to the centered rewards, a method we term ``LSC-transformation'' (log-sigmoid-centered transformation). This transformation has two important properties. First, it emphasizes improving poorly-performing outputs, rather than outputs that already score well. This mitigates both underfitting (where some prompts are not improved) and reward hacking (where the model learns to exploit misspecification of the reward model). Second, it enables principled aggregation of rewards by linking summation to logical conjunction: the sum of transformed rewards corresponds to the probability that the output is ``good'' in all measured properties, in a sense we make precise. Experiments aligning language models to be both helpful and harmless using RLHF show substantial improvements over the baseline (non-transformed) approach.},
  archiveprefix = {arXiv},
  langid = {english},
  keywords = {Computer Science - Artificial Intelligence,Computer Science - Computation and Language},
  file = {/Users/boreas/Zotero/storage/B7A3TDQA/Wang et al. - 2024 - Transforming and Combining Rewards for Aligning Large Language Models.pdf}
}

@article{Wu-FineGrainedHumanFeedbackGivesBetter,
  title = {Fine-{{Grained Human Feedback Gives Better Rewards}} for {{Language Model Training}}},
  author = {Wu, Zeqiu and Hu, Yushi and Shi, Weijia and Dziri, Nouha and Suhr, Alane and Ammanabrolu, Prithviraj and Smith, Noah A and Ostendorf, Mari and Hajishirzi, Hannaneh},
  abstract = {Language models (LMs) often exhibit undesirable text generation behaviors, including generating false, toxic, or irrelevant outputs. Reinforcement learning from human feedback (RLHF)---where human preference judgments on LM outputs are transformed into a learning signal---has recently shown promise in addressing these issues. However, such holistic feedback conveys limited information on long text outputs; it does not indicate which aspects of the outputs influenced user preference; e.g., which parts contain what type(s) of errors. In this paper, we use fine-grained human feedback (e.g., which sentence is false, which sub-sentence is irrelevant) as an explicit training signal. We introduce FINE-GRAINED RLHF, a framework that enables training and learning from reward functions that are fine-grained in two respects: (1) density, providing a reward after every segment (e.g., a sentence) is generated; and (2) incorporating multiple reward models associated with different feedback types (e.g., factual incorrectness, irrelevance, and information incompleteness). We conduct experiments on detoxification and long-form question answering to illustrate how learning with such reward functions leads to improved performance, supported by both automatic and human evaluation. Additionally, we show that LM behaviors can be customized using different combinations of fine-grained reward models. We release all data, collected human feedback, and codes at https://FineGrainedRLHF.github.io.},
  langid = {english},
  file = {/Users/boreas/Zotero/storage/4JP2GXVJ/Wu et al. - Fine-Grained Human Feedback Gives Better Rewards for Language Model Training.pdf}
}

@misc{Xu2024-PerfectBlendRedefiningRLHFMixture,
  title = {The {{Perfect Blend}}: {{Redefining RLHF}} with {{Mixture}} of {{Judges}}},
  shorttitle = {The {{Perfect Blend}}},
  author = {Xu, Tengyu and Helenowski, Eryk and Sankararaman, Karthik Abinav and Jin, Di and Peng, Kaiyan and Han, Eric and Nie, Shaoliang and Zhu, Chen and Zhang, Hejia and Zhou, Wenxuan and Zeng, Zhouhao and He, Yun and Mandyam, Karishma and Talabzadeh, Arya and Khabsa, Madian and Cohen, Gabriel and Tian, Yuandong and Ma, Hao and Wang, Sinong and Fang, Han},
  year = {2024},
  month = sep,
  number = {arXiv:2409.20370},
  eprint = {2409.20370},
  primaryclass = {cs},
  publisher = {arXiv},
  urldate = {2024-10-13},
  abstract = {Reinforcement learning from human feedback (RLHF) has become the leading approach for fine-tuning large language models (LLM). However, RLHF has limitations in multi-task learning (MTL) due to challenges of reward hacking and extreme multi-objective optimization (i.e., trade-off of multiple and/or sometimes conflicting objectives). Applying RLHF for MTL currently requires careful tuning of the weights for reward model and data combinations. This is often done via human intuition and does not generalize. In this work, we introduce a novel post-training paradigm which we called Constrained Generative Policy Optimization (CGPO). The core of CGPO is Mixture of Judges (MoJ) with cost-efficient constrained policy optimization with stratification, which can identify the perfect blend in RLHF in a principled manner. It shows strong empirical results with theoretical guarantees, does not require extensive hyper-parameter tuning, and is plug-and-play in common post-training pipelines. Together, this can detect and mitigate reward hacking behaviors while reaching a pareto-optimal point across an extremely large number of objectives. Our empirical evaluations demonstrate that CGPO significantly outperforms standard RLHF algorithms like PPO and DPO across various tasks including general chat, STEM questions, instruction following, and coding. Specifically, CGPO shows improvements of 7.4\% in AlpacaEval-2 (general chat), 12.5\% in Arena-Hard (STEM \& reasoning), and consistent gains in other domains like math and coding. Notably, PPO, while commonly used, is prone to severe reward hacking in popular coding benchmarks, which CGPO successfully addresses. This breakthrough in RLHF not only tackles reward hacking and extreme multi-objective optimization challenges but also advances the state-of-the-art in aligning general-purpose LLMs for diverse applications.},
  archiveprefix = {arXiv},
  langid = {english},
  keywords = {Computer Science - Artificial Intelligence,Computer Science - Computation and Language,Computer Science - Machine Learning},
  file = {/Users/boreas/Zotero/storage/4EYNGBRB/Xu et al. - 2024 - The Perfect Blend Redefining RLHF with Mixture of Judges.pdf}
}

@misc{Yan2024-RewardRobustRLHFLLMs,
  title = {Reward-{{Robust RLHF}} in {{LLMs}}},
  author = {Yan, Yuzi and Lou, Xingzhou and Li, Jialian and Zhang, Yiping and Xie, Jian and Yu, Chao and Wang, Yu and Yan, Dong and Shen, Yuan},
  year = {2024},
  month = sep,
  number = {arXiv:2409.15360},
  eprint = {2409.15360},
  primaryclass = {cs},
  publisher = {arXiv},
  urldate = {2024-10-13},
  abstract = {As Large Language Models (LLMs) continue to progress toward more advanced forms of intelligence, Reinforcement Learning from Human Feedback (RLHF) is increasingly seen as a key pathway toward achieving Artificial General Intelligence (AGI). However, the reliance on reward-model-based (RM-based) alignment methods introduces significant challenges due to the inherent instability and imperfections of Reward Models (RMs), which can lead to critical issues such as reward hacking and misalignment with human intentions. In this paper, we introduce a reward-robust RLHF framework aimed at addressing these fundamental challenges, paving the way for more reliable and resilient learning in LLMs. Our approach introduces a novel optimization objective that carefully balances performance and robustness by incorporating Bayesian Reward Model Ensembles (BRME) to model the uncertainty set of reward functions. This allows the framework to integrate both nominal performance and minimum reward signals, ensuring more stable learning even with imperfect reward models. Empirical results demonstrate that our framework consistently outperforms traditional RLHF across diverse benchmarks, showing improved accuracy and long-term stability. We also provide a theoretical analysis, demonstrating that reward-robust RLHF approaches the stability of constant reward settings, which proves to be effective in a stochastic-case analysis. Together, these contributions highlight the framework's potential to enhance both the performance and stability of LLM alignment with RLHF.},
  archiveprefix = {arXiv},
  langid = {english},
  keywords = {Computer Science - Artificial Intelligence,Computer Science - Computation and Language,Computer Science - Machine Learning},
  file = {/Users/boreas/Zotero/storage/MSZEZGQW/Yan et al. - 2024 - Reward-Robust RLHF in LLMs.pdf}
}

@misc{Yuan2024-AdvancingLLMReasoningGeneralistsPreference,
  title = {Advancing {{LLM Reasoning Generalists}} with {{Preference Trees}}},
  author = {Yuan, Lifan and Cui, Ganqu and Wang, Hanbin and Ding, Ning and Wang, Xingyao and Deng, Jia and Shan, Boji and Chen, Huimin and Xie, Ruobing and Lin, Yankai and Liu, Zhenghao and Zhou, Bowen and Peng, Hao and Liu, Zhiyuan and Sun, Maosong},
  year = {2024},
  month = apr,
  number = {arXiv:2404.02078},
  eprint = {2404.02078},
  primaryclass = {cs},
  publisher = {arXiv},
  urldate = {2024-10-13},
  abstract = {We introduce Eurus, a suite of large language models (LLMs) optimized for reasoning. Finetuned from Mistral-7B and CodeLlama-70B, Eurus models achieve state-of-the-art results among open-source models on a diverse set of benchmarks covering mathematics, code generation, and logical reasoning problems. Notably, Eurus-70B beats GPT-3.5 Turbo in reasoning through a comprehensive benchmarking across 12 tests covering five tasks, and achieves a 33.3\% pass@1 accuracy on LeetCode and 32.6\% on TheoremQA, two challenging benchmarks, substantially outperforming existing open-source models by margins more than 13.3\%. The strong performance of Eurus can be primarily attributed to UltraInteract, our newly-curated large-scale, high-quality alignment dataset specifically designed for complex reasoning tasks. UltraInteract can be used in both supervised fine-tuning and preference learning. For each instruction, it includes a preference tree consisting of (1) reasoning chains with diverse planning strategies in a unified format, (2) multi-turn interaction trajectories with the environment and the critique, and (3) pairwise data to facilitate preference learning. UltraInteract allows us to conduct an in-depth exploration of preference learning for reasoning tasks. Our investigation reveals that some well-established preference learning algorithms may be less suitable for reasoning tasks compared to their effectiveness in general conversations. Inspired by this, we derive a novel reward modeling objective which, together with UltraInteract, leads to a strong reward model.},
  archiveprefix = {arXiv},
  langid = {english},
  keywords = {Computer Science - Artificial Intelligence,Computer Science - Computation and Language,Computer Science - Machine Learning},
  file = {/Users/boreas/Zotero/storage/RDWKZQLT/Yuan et al. - 2024 - Advancing LLM Reasoning Generalists with Preference Trees.pdf}
}

@inproceedings{Zhou2024-OnePreferenceFitsAllAlignmentMultiObjectiveDirectPreference,
  title = {Beyond {{One-Preference-Fits-All Alignment}}: {{Multi-Objective Direct Preference Optimization}}},
  shorttitle = {Beyond {{One-Preference-Fits-All Alignment}}},
  booktitle = {Findings of the {{Association}} for {{Computational Linguistics ACL}} 2024},
  author = {Zhou, Zhanhui and Liu, Jie and Shao, Jing and Yue, Xiangyu and Yang, Chao and Ouyang, Wanli and Qiao, Yu},
  editor = {Ku, Lun-Wei and Martins, Andre and Srikumar, Vivek},
  year = {2024},
  month = aug,
  pages = {10586--10613},
  publisher = {Association for Computational Linguistics},
  address = {Bangkok, Thailand and virtual meeting},
  doi = {10.18653/v1/2024.findings-acl.630},
  urldate = {2024-10-09},
  abstract = {A single language model, even when aligned with labelers through reinforcement learning from human feedback (RLHF), may not suit all human preferences. Recent approaches therefore prefer customization, gathering multi-dimensional feedback, and creating distinct reward models for each dimension.Different language models are then optimized for various preferences using multi-objective RLHF (MORLHF) with varying reward weights.However, RL fine-tuning is unstable and resource-heavy, especially with diverse and usually conflicting objectives.In this paper, we present Multi-Objective Direct Preference Optimization (MODPO), an RL-free extension of Direct Preference Optimization (DPO) for multiple alignment objectives.Essentially, MODPO folds language modeling directly into reward modeling, training language models as implicit collective reward models that combine all objectives with specific weights. MODPO theoretically yields the same optimal solutions as MORLHF but is practically more stable and efficient.Empirical results in safety alignment and long-form question answering show that MODPO matches or outperforms existing methods, producing a Pareto front of language models catering to diverse preferences with three times less computational resources compared to MORLHF.Code is available at https://github.com/ZHZisZZ/modpo.},
  file = {/Users/boreas/Zotero/storage/ILGPHJKX/ZhouLiuShao2024-Beyond One-Preference-Fits-All Alignment.pdf}
}
