September 9, 2024 Dataset Restricted Access

Völschow, Marcel; Buczek, P.; Carreno-Mosquera, P.; Mousavias, C.; Reganova, S.; Roldan-Rodriguez, E.; Steinbach, Peter; Strube, A.

JSON Export

{
  "doi": "10.14278/rodare.3137", 
  "updated": "2024-09-12T09:24:19.933452+00:00", 
  "stats": {
    "volume": 0.0, 
    "unique_downloads": 0.0, 
    "version_unique_downloads": 0.0, 
    "unique_views": 8.0, 
    "downloads": 0.0, 
    "version_unique_views": 8.0, 
    "version_views": 12.0, 
    "version_downloads": 0.0, 
    "version_volume": 0.0, 
    "views": 12.0
  }, 
  "id": 3137, 
  "conceptdoi": "10.14278/rodare.3136", 
  "links": {
    "badge": "https://rodare.hzdr.de/badge/doi/10.14278/rodare.3137.svg", 
    "doi": "https://doi.org/10.14278/rodare.3137", 
    "conceptbadge": "https://rodare.hzdr.de/badge/doi/10.14278/rodare.3136.svg", 
    "conceptdoi": "https://doi.org/10.14278/rodare.3136", 
    "html": "https://rodare.hzdr.de/record/3137", 
    "latest": "https://rodare.hzdr.de/api/records/3137", 
    "latest_html": "https://rodare.hzdr.de/record/3137"
  }, 
  "conceptrecid": "3136", 
  "revision": 4, 
  "created": "2024-09-10T09:37:15.570547+00:00", 
  "metadata": {
    "doi": "10.14278/rodare.3137", 
    "description": "<p>Large-Language Models such as ChatGPT have the potential to revo-<br>\r\nlutionize academic teaching in physics in a similar way the electronic calculator,<br>\r\nthe home computer or the internet did. AI models are patient, produce answers<br>\r\ntailored to a student\u2019s needs and are accessible whenever needed. Those involved<br>\r\nin academic teaching are facing a number of questions: Just how reliable are pub-<br>\r\nlicly accessible models in answering, how does the question\u2019s language affect the<br>\r\nmodels\u2019 performance and how well do the models perform with more difficult tasks<br>\r\nbeyond retrieval? To adress these questions, we benchmark a number of publicly<br>\r\navailable models on the mlphys101 dataset, a new set of 823 university level MC5<br>\r\nquestions and answers released alongside this work. While the original questions<br>\r\nare in English, we employ GPT-4 to translate them into various other languages,<br>\r\nfollowed by revision and refinement by native speakers. Our findings indicate that<br>\r\nstate-of-the-art models perform well on questions involving the replication of facts,<br>\r\ndefinitions, and basic concepts, but struggle with multi-step quantitative reason-<br>\r\ning. This aligns with existing literature that highlights the challenges LLMs face<br>\r\nin mathematical and logical reasoning tasks. We conclude that the most advanced<br>\r\ncurrent LLMs are a valuable addition to the academic curriculum and LLM pow-<br>\r\nered translations are a viable method to increase the accessibility of materials, but<br>\r\ntheir utility for more difficult quantitative tasks remains limited.</p>\r\n\r\n<p>The dataset is available in English here only and will be removed, once the mlphys101 publication was accepted and released to the public.</p>", 
    "related_identifiers": [
      {
        "identifier": "https://www.hzdr.de/publications/Publ-39561", 
        "relation": "isIdenticalTo", 
        "scheme": "url"
      }, 
      {
        "identifier": "10.14278/rodare.3136", 
        "relation": "isVersionOf", 
        "scheme": "doi"
      }
    ], 
    "access_right_category": "danger", 
    "title": "mlphys101 - Exploring the performance of Large-Language Models in multilingual undergraduate physics education", 
    "publication_date": "2024-09-09", 
    "access_right": "restricted", 
    "communities": [
      {
        "id": "rodare"
      }
    ], 
    "relations": {
      "version": [
        {
          "index": 0, 
          "last_child": {
            "pid_type": "recid", 
            "pid_value": "3137"
          }, 
          "count": 1, 
          "parent": {
            "pid_type": "recid", 
            "pid_value": "3136"
          }, 
          "is_last": true
        }
      ]
    }, 
    "access_conditions": "<p>Access will be granted based on reasonable request until we wait for the publication by the original author of the dataset on another platform.</p>", 
    "resource_type": {
      "type": "dataset", 
      "title": "Dataset"
    }, 
    "doc_id": "1", 
    "notes": "The dataset is available in English here only and will be removed, once the mlphys101 publication was accepted and released to the public.", 
    "pub_id": "39561", 
    "language": "eng", 
    "creators": [
      {
        "name": "V\u00f6lschow, Marcel"
      }, 
      {
        "name": "Buczek, P."
      }, 
      {
        "name": "Carreno-Mosquera, P."
      }, 
      {
        "name": "Mousavias, C."
      }, 
      {
        "name": "Reganova, S."
      }, 
      {
        "name": "Roldan-Rodriguez, E."
      }, 
      {
        "orcid": "0000-0002-4974-230X", 
        "name": "Steinbach, Peter"
      }, 
      {
        "name": "Strube, A."
      }
    ], 
    "keywords": [
      "machine learning", 
      "deep learning", 
      "large language models", 
      "chatgpt", 
      "blablador"
    ]
  }, 
  "owners": [
    156
  ]
}